@dxos/echo-db 2.33.9-dev.6efa8ddf → 2.33.9-dev.845d4f3e

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dxos/echo-db",
-  "version": "2.33.9-dev.6efa8ddf",
+  "version": "2.33.9-dev.845d4f3e",
   "description": "ECHO database.",
   "license": "MIT",
   "main": "dist/src/index.js",
@@ -13,22 +13,22 @@
     "src"
   ],
   "dependencies": {
-    "@dxos/async": "2.33.9-dev.6efa8ddf",
-    "@dxos/codec-protobuf": "2.33.9-dev.6efa8ddf",
-    "@dxos/credentials": "2.33.9-dev.6efa8ddf",
-    "@dxos/crypto": "2.33.9-dev.6efa8ddf",
-    "@dxos/debug": "2.33.9-dev.6efa8ddf",
-    "@dxos/echo-protocol": "2.33.9-dev.6efa8ddf",
-    "@dxos/feed-store": "2.33.9-dev.6efa8ddf",
-    "@dxos/mesh-protocol": "2.33.9-dev.6efa8ddf",
-    "@dxos/model-factory": "2.33.9-dev.6efa8ddf",
-    "@dxos/network-manager": "2.33.9-dev.6efa8ddf",
-    "@dxos/object-model": "2.33.9-dev.6efa8ddf",
-    "@dxos/protocol-plugin-presence": "2.33.9-dev.6efa8ddf",
-    "@dxos/protocol-plugin-replicator": "2.33.9-dev.6efa8ddf",
-    "@dxos/protocols": "2.33.9-dev.6efa8ddf",
-    "@dxos/random-access-multi-storage": "2.33.9-dev.6efa8ddf",
-    "@dxos/util": "2.33.9-dev.6efa8ddf",
+    "@dxos/async": "2.33.9-dev.845d4f3e",
+    "@dxos/codec-protobuf": "2.33.9-dev.845d4f3e",
+    "@dxos/credentials": "2.33.9-dev.845d4f3e",
+    "@dxos/crypto": "2.33.9-dev.845d4f3e",
+    "@dxos/debug": "2.33.9-dev.845d4f3e",
+    "@dxos/echo-protocol": "2.33.9-dev.845d4f3e",
+    "@dxos/feed-store": "2.33.9-dev.845d4f3e",
+    "@dxos/mesh-protocol": "2.33.9-dev.845d4f3e",
+    "@dxos/model-factory": "2.33.9-dev.845d4f3e",
+    "@dxos/network-manager": "2.33.9-dev.845d4f3e",
+    "@dxos/object-model": "2.33.9-dev.845d4f3e",
+    "@dxos/protocol-plugin-presence": "2.33.9-dev.845d4f3e",
+    "@dxos/protocol-plugin-replicator": "2.33.9-dev.845d4f3e",
+    "@dxos/protocols": "2.33.9-dev.845d4f3e",
+    "@dxos/random-access-multi-storage": "2.33.9-dev.845d4f3e",
+    "@dxos/util": "2.33.9-dev.845d4f3e",
     "assert": "^2.0.0",
     "base-x": "~3.0.9",
     "buffer-json-encoding": "^1.0.2",

package/src/api/index.ts

@@ -4,3 +4,4 @@
 
 export * from './result-set';
 export * from './schema';
+export * from './subscription';

package/src/api/subscription.ts

@@ -0,0 +1,24 @@
+//
+// Copyright 2021 DXOS.org
+//
+
+import { Event } from '@dxos/async';
+import { Stream } from '@dxos/codec-protobuf';
+
+import { ResultSet } from './result-set';
+
+export const resultSetToStream = <T, U>(resultSet: ResultSet<T>, map: (arg: T[]) => U): Stream<U> => new Stream(({ next }) => {
+  next(map(resultSet.value));
+  return resultSet.update.on(() => next(map(resultSet.value)));
+});
+
+export const streamToResultSet = <T, U>(stream: Stream<T>, map: (arg?: T) => U[]): ResultSet<U> => {
+  const event = new Event();
+  let lastItem: T | undefined;
+  stream.subscribe(data => {
+    lastItem = data;
+    event.emit();
+  });
+
+  return new ResultSet(event, () => map(lastItem));
+};

package/src/echo-api.test.ts

@@ -0,0 +1,162 @@
+//
+// Copyright 2020 DXOS.org
+//
+
+import expect from 'expect';
+import { it as test } from 'mocha';
+
+// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Proxy
+// https://javascript.info/proxy
+// https://www.javascripttutorial.net/es6/javascript-proxy
+
+// TODO(burdon): Array-specific methods (e.g., push, replace, etc.)
+//  http://www.jomendez.com/2018/12/29/querying-arrays-with-more-readable-methods-using-javascript-proxy
+// class ArrayHandler<T> extends Handler {}
+
+type Mutation = [property: string, value: any]
+
+interface Object {
+  // Special properties added by proto generator.
+  $id?: string
+  $type?: string
+  $mutations?: Mutation[]
+}
+
+class Handler<T> {
+  _mutations: Mutation[] = [];
+
+  constructor (
+    private readonly _parent?: Handler<any>,
+    protected readonly _property?: string,
+    protected readonly _callback?: (value: T) => void
+  ) {}
+
+  get mutations () {
+    return this._mutations;
+  }
+
+  addMutation (mutation: Mutation) {
+    if (this._parent) {
+      this._parent.addMutation(mutation);
+    } else {
+      this._mutations.push(mutation);
+    }
+  }
+
+  getMeta (p: string) {
+    switch (p) {
+      case '$mutations': {
+        return this._mutations;
+      }
+    }
+  }
+}
+
+//
+// TODO(burdon): Generated from proto.
+//
+
+interface Contact extends Object {
+  name?: {
+    first?: string
+    last?: string
+  }
+  age?: number
+  emails?: string[]
+
+  fullname?: string // TODO(burdon): Show how to add custom functions.
+}
+
+class ContactNameHandler extends Handler<Contact['name']> {
+  get (obj: any, p: string) {
+    switch (p) {
+      case 'first':
+      case 'last': {
+        return obj[p];
+      }
+    }
+  }
+
+  set (obj: any, p: string, value: any) {
+    switch (p) {
+      case 'first':
+      case 'last': {
+        this.addMutation([`${this._property}.${p}`, value]);
+        obj[p] = value;
+        this._callback?.(obj);
+        return true;
+      }
+    }
+
+    return false;
+  }
+}
+
+class ContactHandler extends Handler<Contact> {
+  get (obj: any, p: string) {
+    const result = super.getMeta(p);
+    if (result !== undefined) {
+      return result;
+    }
+
+    switch (p) {
+      case 'name': {
+        return new Proxy(obj[p] ?? {}, new ContactNameHandler(this, p, value => {
+          obj[p] = value;
+        }));
+      }
+    }
+
+    return obj[p];
+  }
+
+  set (obj: any, p: string, value: any) {
+    this.addMutation([p, value]);
+    obj[p] = value;
+    return true; // Throws TypeError if false.
+  }
+}
+
+type ItemType<T extends object> = [string, () => Handler<T>]
+
+const ContactType: ItemType<Contact> = ['example.Contact', () => new ContactHandler()];
+
+class Database {
+  private readonly _types = new Map<ItemType<any>, (value: any) => typeof Proxy>();
+
+  addType (type: ItemType<any>, constructor: (value: any) => any) {
+    this._types.set(type, constructor);
+    return this;
+  }
+
+  create<T extends object> (type: ItemType<T>, value: T = {} as T): T {
+    const constructor = this._types.get(type)!;
+    return constructor(value) as any;
+  }
+}
+
+describe('API', () => {
+  test('Pseudo-code', () => {
+    const db = new Database();
+    db.addType(ContactType, value => new Proxy<Contact>(value, new ContactHandler()));
+
+    // Create type-safe proxy.
+    const item = db.create(ContactType);
+
+    item.age = 64;
+    expect(item.$mutations).toHaveLength(1);
+    expect(item).toEqual({ age: 64 });
+
+    item.name!.first = 'Alice';
+    expect(item.name?.first).toBe('Alice');
+    expect(item.$mutations).toHaveLength(2);
+    expect(item).toEqual({ age: 64, name: { first: 'Alice' } });
+
+    expect(item.$mutations).toStrictEqual([
+      ['age', 64],
+      ['name.first', 'Alice']
+    ]);
+
+    // TODO(burdon): Get item id, state, mutations, etc.
+  });
+});

package/src/echo.test.ts

@@ -31,8 +31,8 @@ describe('ECHO', () => {
 
   const setup = async ({ createProfile, displayName }: SetupOptions = {}) => {
     const echo = new ECHO();
-
     await echo.open();
+
     afterTest(() => echo.close());
 
     if (createProfile) {

package/src/echo.ts

@@ -25,7 +25,7 @@ import { InvitationDescriptor, OfflineInvitationClaimer } from './invitations';
 import { DataServiceRouter } from './packlets/database';
 import { IdentityNotInitializedError, InvalidStorageVersionError } from './packlets/errors';
 import { OpenProgress, PartyFactory, DataParty, PartyManager } from './parties';
-import { MetadataStore, STORAGE_VERSION, PartyFeedProvider } from './pipeline';
+import { STORAGE_VERSION, MetadataStore, PartyFeedProvider } from './pipeline';
 import { SnapshotStore } from './snapshots';
 
 const log = debug('dxos:echo');
@@ -85,18 +85,19 @@ export interface EchoCreationOptions {
  * Messages are streamed into the pipeline (from the `FeedStore`) in logical order, determined by the
  * `Timeframe` (which implements a vector clock).
  */
-// TODO(burdon): Create ECHOError class for public errors.
 export class ECHO {
-  private readonly _halo: HALO;
   private readonly _keyring: Keyring;
 
+  // TODO(burdon): Factor out.
+  private readonly _halo: HALO;
+  private readonly _partyManager: PartyManager;
+  private readonly _subscriptions = new SubscriptionGroup();
+
   private readonly _storage: Storage;
   private readonly _feedStore: FeedStore;
   private readonly _modelFactory: ModelFactory;
   private readonly _networkManager: NetworkManager;
   private readonly _snapshotStore: SnapshotStore;
-  private readonly _partyManager: PartyManager;
-  private readonly _subs = new SubscriptionGroup();
   private readonly _metadataStore: MetadataStore;
   private readonly _dataServiceRouter: DataServiceRouter;
 
@@ -153,7 +154,7 @@ export class ECHO {
     this._partyManager = new PartyManager(
       this._metadataStore,
       this._snapshotStore,
-      () => this.halo.identity,
+      () => this._halo.identity,
       partyFactory
     );
 
@@ -170,7 +171,7 @@ export class ECHO {
     this._halo.identityReady.once(() => {
       // It might be the case that halo gets closed before this has a chance to execute.
       if (this.halo.identity?.halo.isOpen) {
-        this._subs.push(autoPartyOpener(this.halo.identity.preferences!, this._partyManager));
+        this._subscriptions.push(autoPartyOpener(this.halo.identity.preferences!, this._partyManager));
       }
     });
 
@@ -260,7 +261,7 @@ export class ECHO {
       return;
     }
 
-    this._subs.unsubscribe();
+    this._subscriptions.unsubscribe();
 
     await this.halo.close();
 
@@ -272,7 +273,6 @@ export class ECHO {
 
   /**
    * Removes all data and closes this ECHO instance.
-   *
    * The instance will be in an unusable state at this point and a page refresh is recommended.
    */
   // TODO(burdon): Enable re-open.
@@ -371,7 +371,8 @@ export class ECHO {
     assert(this._partyManager.isOpen, new InvalidStateError());
 
     const actualSecretProvider =
-      secretProvider ?? OfflineInvitationClaimer.createSecretProvider(this.halo.identity?.createCredentialsSigner() ?? raise(new IdentityNotInitializedError()));
+      secretProvider ?? OfflineInvitationClaimer.createSecretProvider(this.halo.identity?.createCredentialsSigner() ??
+        raise(new IdentityNotInitializedError()));
 
     return this._partyManager.joinParty(invitationDescriptor, actualSecretProvider);
   }

package/src/halo/halo-party.ts

@@ -28,9 +28,9 @@ export const HALO_PARTY_DEVICE_PREFERENCES_TYPE = 'dxos:item/halo/device/prefere
 /**
  * A record in HALO party representing a party that user is currently a member of.
  */
-export interface JoinedParty {
-  partyKey: PublicKey,
-  genesisFeed: PublicKey,
+export type JoinedParty = {
+  partyKey: PublicKey
+  genesisFeed: PublicKey
 }
 
 /**

package/src/halo/halo.test.ts

@@ -5,9 +5,7 @@
 import expect from 'expect';
 import { it as test } from 'mocha';
 
-import {
-  defaultSecretProvider, generateSeedPhrase, keyPairFromSeedPhrase, Keyring
-} from '@dxos/credentials';
+import { defaultSecretProvider, generateSeedPhrase, keyPairFromSeedPhrase, Keyring } from '@dxos/credentials';
 import { codec } from '@dxos/echo-protocol';
 import { FeedStore } from '@dxos/feed-store';
 import { ModelFactory } from '@dxos/model-factory';

package/src/halo/identity-manager.ts

@@ -39,15 +39,10 @@ export class IdentityManager {
     assert(halo.isOpen, 'HALO must be open.');
 
     // Wait for the minimum set of keys and messages we need for proper function:
-    //
     // - KeyAdmit message for the current device so we can build the device KeyChain.
     // - Identity genesis so it can be copied into newly joined parties.
-    //
     const deviceKey = this._keyring.findKey(Keyring.signingFilter({ type: KeyType.DEVICE })) ?? failUndefined();
-    await waitForCondition(() =>
-      halo.processor.isMemberKey(deviceKey.publicKey) &&
-      halo.identityGenesis
-    );
+    await waitForCondition(() => halo.processor.isMemberKey(deviceKey.publicKey) && halo.identityGenesis);
 
     this._identity = new Identity(this._keyring, halo);
     this.ready.emit();

package/src/halo/identity.ts

@@ -27,6 +27,7 @@ export class Identity implements IdentityCredentials {
   private readonly _deviceKeyChain: KeyChain;
 
   /**
+   * @param _keyring
    * @param _halo HALO party. Must be open.
    */
   constructor (
@@ -86,6 +87,7 @@ export class Identity implements IdentityCredentials {
   /**
    * HALO party. Must be open.
    */
+  // TODO(burdon): Remove.
   get halo (): HaloParty {
     return this._halo;
   }
@@ -100,8 +102,6 @@ export class Identity implements IdentityCredentials {
   }
 }
 
-export type IdentityProvider = () => Identity | undefined;
-
 const getDeviceKeyChainFromHalo = (halo: HaloParty, deviceKey: KeyRecord) => {
   try {
     return Keyring.buildKeyChain(

package/src/halo/preferences.ts

@@ -139,6 +139,7 @@ export class Preferences {
     if (!database) {
       return;
     }
+
     const [partyDesc] = database
       .select({ type: HALO_PARTY_DESCRIPTOR_TYPE })
       .filter(partyMarker => joinedParty.partyKey.equals(partyMarker.model.get('publicKey')))

package/src/packlets/database/item-manager.ts

@@ -26,14 +26,14 @@ export interface ModelConstructionOptions {
 }
 
 export interface ItemConstructionOptions extends ModelConstructionOptions {
-  itemType: ItemType | undefined,
-  parentId?: ItemID,
+  itemType: ItemType | undefined
+  parentId?: ItemID
 }
 
 export interface LinkConstructionOptions extends ModelConstructionOptions {
-  itemType: ItemType | undefined,
-  source: ItemID;
-  target: ItemID;
+  itemType: ItemType | undefined
+  source: ItemID
+  target: ItemID
 }
 
 /**

package/src/parties/data-party.test.ts

@@ -5,7 +5,9 @@
 import expect from 'expect';
 import { it as test } from 'mocha';
 
-import { createKeyAdmitMessage, createPartyGenesisMessage, defaultSecretProvider, Keyring, KeyType, codec as haloCodec } from '@dxos/credentials';
+import {
+  createKeyAdmitMessage, createPartyGenesisMessage, defaultSecretProvider, Keyring, KeyType, codec as haloCodec
+} from '@dxos/credentials';
 import { codec } from '@dxos/echo-protocol';
 import { FeedStore } from '@dxos/feed-store';
 import { ModelFactory } from '@dxos/model-factory';

package/src/parties/data-party.ts

@@ -30,7 +30,7 @@ export const PARTY_TITLE_PROPERTY = 'title'; // TODO(burdon): Remove (should not
 
 // TODO(burdon): Factor out public API.
 export interface PartyMember {
-  publicKey: PublicKey,
+  publicKey: PublicKey
   displayName?: string
 }
 
@@ -142,7 +142,7 @@ export class DataParty {
     await this._preferences?.setLastKnownTitle(title);
   }
 
-  get genesisFeedKey () {
+  get genesisFeedKey (): PublicKey {
     assert(this._genesisFeedKey);
     return this._genesisFeedKey;
   }

package/src/parties/party-factory.ts

@@ -19,7 +19,7 @@ import { ModelFactory } from '@dxos/model-factory';
 import { NetworkManager } from '@dxos/network-manager';
 import { ObjectModel } from '@dxos/object-model';
 import { PublicKey, Timeframe } from '@dxos/protocols';
-import { humanize } from '@dxos/util';
+import { humanize, Provider } from '@dxos/util';
 
 import {
   createDataPartyAdmissionMessages,
@@ -27,7 +27,7 @@ import {
 } from '../invitations';
 import { IdentityNotInitializedError } from '../packlets/errors';
 import { MetadataStore, PartyFeedProvider, PipelineOptions } from '../pipeline';
-import { IdentityCredentialsProvider } from '../protocol/identity-credentials';
+import { IdentityCredentials } from '../protocol/identity-credentials';
 import { SnapshotStore } from '../snapshots';
 import { DataParty, PARTY_ITEM_TYPE } from './data-party';
 
@@ -38,7 +38,7 @@ const log = debug('dxos:echo-db:party-factory');
  */
 export class PartyFactory {
   constructor (
-    private readonly _identityProvider: IdentityCredentialsProvider,
+    private readonly _identityProvider: Provider<IdentityCredentials | undefined>,
     private readonly _networkManager: NetworkManager,
     private readonly _modelFactory: ModelFactory,
     private readonly _snapshotStore: SnapshotStore,

package/src/parties/party-manager.ts

@@ -11,11 +11,11 @@ import { SecretProvider } from '@dxos/credentials';
 import { failUndefined, timed } from '@dxos/debug';
 import { PartyKey, PartySnapshot } from '@dxos/echo-protocol';
 import { PublicKey } from '@dxos/protocols';
-import { ComplexMap, boolGuard } from '@dxos/util';
+import { ComplexMap, boolGuard, Provider } from '@dxos/util';
 
 import { InvitationDescriptor } from '../invitations';
 import { MetadataStore } from '../pipeline';
-import { IdentityCredentialsProvider } from '../protocol/identity-credentials';
+import { IdentityCredentials } from '../protocol/identity-credentials';
 import { SnapshotStore } from '../snapshots';
 import { DataParty, PARTY_ITEM_TYPE, PARTY_TITLE_PROPERTY } from './data-party';
 import { PartyFactory } from './party-factory';
@@ -25,9 +25,9 @@ export const CONTACT_DEBOUNCE_INTERVAL = 500;
 const log = debug('dxos:echo-db:party-manager');
 
 export interface OpenProgress {
-  haloOpened: boolean;
-  partiesOpened?: number;
-  totalParties?: number;
+  haloOpened: boolean
+  partiesOpened?: number
+  totalParties?: number
 }
 
 /**
@@ -42,16 +42,12 @@ export class PartyManager {
   // Map of parties by party key.
   private readonly _parties = new ComplexMap<PublicKey, DataParty>(key => key.toHex());
 
-  // Unsubscribe handlers.
-  // TODO(burdon): Never used.
-  private readonly _onCloseHandlers: (() => void)[] = [];
-
   private _open = false;
 
   constructor (
     private readonly _metadataStore: MetadataStore,
     private readonly _snapshotStore: SnapshotStore,
-    private readonly _identityProvider: IdentityCredentialsProvider,
+    private readonly _identityProvider: Provider<IdentityCredentials | undefined>,
     private readonly _partyFactory: PartyFactory
   ) {}
 
@@ -129,9 +125,6 @@ export class PartyManager {
     }
     this._open = false;
 
-    // Clean-up.
-    this._onCloseHandlers.forEach(callback => callback());
-
     // Close parties.
     for (const party of this._parties.values()) {
       if (party.isOpen) {

package/src/parties/party-preferences.ts

@@ -4,12 +4,12 @@
 
 import assert from 'node:assert';
 
-import { PARTY_TITLE_PROPERTY, DataParty } from '.';
 import { Preferences } from '../halo/preferences';
+import { PARTY_TITLE_PROPERTY, DataParty } from './data-party';
 
 export interface ActivationOptions {
-  global?: boolean;
-  device?: boolean;
+  global?: boolean
+  device?: boolean
 }
 
 /**

package/src/pipeline/party-pipeline.test.ts

@@ -18,9 +18,10 @@ import { PublicKey, Timeframe } from '@dxos/protocols';
 import { createStorage, StorageType } from '@dxos/random-access-multi-storage';
 import { afterTest } from '@dxos/testutils';
 
-import { MetadataStore, PartyFeedProvider } from '.';
-import { createReplicatorPlugin } from '../protocol/replicator-plugin';
+import { createReplicatorPlugin } from '../protocol';
 import { SnapshotStore } from '../snapshots';
+import { MetadataStore } from './metadata-store';
+import { PartyFeedProvider } from './party-feed-provider';
 import { PartyPipeline } from './party-pipeline';
 
 describe('PartyPipeline', () => {

package/src/pipeline/party-pipeline.ts

@@ -14,20 +14,23 @@ import { ModelFactory } from '@dxos/model-factory';
 import { PublicKey, Timeframe } from '@dxos/protocols';
 import { SubscriptionGroup } from '@dxos/util';
 
-import { createMessageSelector, PartyProcessor, PartyFeedProvider, FeedMuxer } from '.';
 import { Database, FeedDatabaseBackend, TimeframeClock } from '../packlets/database';
 import { createAutomaticSnapshots, SnapshotStore } from '../snapshots';
+import { FeedMuxer } from './feed-muxer';
+import { createMessageSelector } from './message-selector';
+import { PartyFeedProvider } from './party-feed-provider';
+import { PartyProcessor } from './party-processor';
 
 const DEFAULT_SNAPSHOT_INTERVAL = 100; // Every 100 messages.
 
 export interface PipelineOptions {
-  readLogger?: (msg: any) => void;
-  writeLogger?: (msg: any) => void;
-  readOnly?: boolean;
+  readLogger?: (msg: any) => void
+  writeLogger?: (msg: any) => void
+  readOnly?: boolean
   // TODO(burdon): Hierarchical options.
   // snapshots: { enabled: true, interval: 100 } }
-  snapshots?: boolean;
-  snapshotInterval?: number;
+  snapshots?: boolean
+  snapshotInterval?: number
 }
 
 export interface OpenOptions {

package/src/protocol/identity-credentials.ts

@@ -27,8 +27,6 @@ export interface IdentityCredentials {
   contacts: ContactManager | undefined
 }
 
-export type IdentityCredentialsProvider = () => IdentityCredentials | undefined
-
 export const createTestIdentityCredentials = async (keyring: Keyring): Promise<IdentityCredentials> => {
   const identityKey = await keyring.createKeyRecord({ type: KeyType.IDENTITY });
   const deviceKey = await keyring.createKeyRecord({ type: KeyType.DEVICE });

package/src/protocol/index.ts

@@ -8,3 +8,4 @@ export * from './auth-plugin';
 export * from './halo-recovery-plugin';
 export * from './offline-invitation-plugin';
 export * from './party-protocol-factory';
+export * from './replicator-plugin';

package/src/protocol/party-protocol-factory.ts

@@ -11,7 +11,7 @@ import { MMSTTopology, NetworkManager, Plugin } from '@dxos/network-manager';
 import { PresencePlugin } from '@dxos/protocol-plugin-presence';
 import { PublicKey } from '@dxos/protocols';
 
-import { CredentialsProvider } from '.';
+import { CredentialsProvider } from './authenticator';
 
 const log = debug('dxos:echo-db:party-protocol-factory');