@dxos/echo-db 2.33.3-dev.1cc72521 → 2.33.3-dev.762b75aa

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dxos/echo-db",
-  "version": "2.33.3-dev.1cc72521",
+  "version": "2.33.3-dev.762b75aa",
   "description": "ECHO database.",
   "license": "MIT",
   "main": "dist/src/index.js",
@@ -10,22 +10,22 @@
     "src"
   ],
   "dependencies": {
-    "@dxos/async": "2.33.3-dev.1cc72521",
-    "@dxos/codec-protobuf": "2.33.3-dev.1cc72521",
-    "@dxos/credentials": "2.33.3-dev.1cc72521",
-    "@dxos/crypto": "2.33.3-dev.1cc72521",
-    "@dxos/debug": "2.33.3-dev.1cc72521",
-    "@dxos/echo-protocol": "2.33.3-dev.1cc72521",
-    "@dxos/feed-store": "2.33.3-dev.1cc72521",
-    "@dxos/mesh-protocol": "2.33.3-dev.1cc72521",
-    "@dxos/model-factory": "2.33.3-dev.1cc72521",
-    "@dxos/network-manager": "2.33.3-dev.1cc72521",
-    "@dxos/object-model": "2.33.3-dev.1cc72521",
-    "@dxos/protocol-plugin-presence": "2.33.3-dev.1cc72521",
-    "@dxos/protocol-plugin-replicator": "2.33.3-dev.1cc72521",
-    "@dxos/protocols": "2.33.3-dev.1cc72521",
-    "@dxos/random-access-multi-storage": "2.33.3-dev.1cc72521",
-    "@dxos/util": "2.33.3-dev.1cc72521",
+    "@dxos/async": "2.33.3-dev.762b75aa",
+    "@dxos/codec-protobuf": "2.33.3-dev.762b75aa",
+    "@dxos/credentials": "2.33.3-dev.762b75aa",
+    "@dxos/crypto": "2.33.3-dev.762b75aa",
+    "@dxos/debug": "2.33.3-dev.762b75aa",
+    "@dxos/echo-protocol": "2.33.3-dev.762b75aa",
+    "@dxos/feed-store": "2.33.3-dev.762b75aa",
+    "@dxos/mesh-protocol": "2.33.3-dev.762b75aa",
+    "@dxos/model-factory": "2.33.3-dev.762b75aa",
+    "@dxos/network-manager": "2.33.3-dev.762b75aa",
+    "@dxos/object-model": "2.33.3-dev.762b75aa",
+    "@dxos/protocol-plugin-presence": "2.33.3-dev.762b75aa",
+    "@dxos/protocol-plugin-replicator": "2.33.3-dev.762b75aa",
+    "@dxos/protocols": "2.33.3-dev.762b75aa",
+    "@dxos/random-access-multi-storage": "2.33.3-dev.762b75aa",
+    "@dxos/util": "2.33.3-dev.762b75aa",
     "base-x": "~3.0.9",
     "buffer-json-encoding": "^1.0.2",
     "debug": "^4.3.3",

package/src/echo.test.ts

@@ -264,8 +264,8 @@ describe('ECHO', () => {
     const a = await setup({ createProfile: true });
     const b = await setup();
 
-    expect(a.halo.isInitialized).toEqual(true);
-    expect(b.halo.isInitialized).toEqual(false);
+    expect(!!a.halo.identity).toEqual(true);
+    expect(!!b.halo.identity).toEqual(false);
 
     expect(a.queryParties().value.length).toBe(0);
     await a.createParty();
@@ -430,11 +430,11 @@ describe('ECHO', () => {
     const b1 = await setup({ createProfile: true });
     const b2 = await setup();
 
-    expect(a1.halo.isInitialized).toBeTruthy();
-    expect(a2.halo.isInitialized).toBeFalsy();
+    expect(a1.halo.identity).toBeTruthy();
+    expect(a2.halo.identity).toBeFalsy();
 
-    expect(b1.halo.isInitialized).toBeTruthy();
-    expect(b2.halo.isInitialized).toBeFalsy();
+    expect(b1.halo.identity).toBeTruthy();
+    expect(b2.halo.identity).toBeFalsy();
 
     await Promise.all([
       (async () => {
@@ -529,13 +529,13 @@ describe('ECHO', () => {
 
     const b = await setup();
 
-    expect(a.halo.isInitialized).toBeTruthy();
-    expect(b.halo.isInitialized).toBeFalsy();
+    expect(a.halo.identity).toBeTruthy();
+    expect(b.halo.identity).toBeFalsy();
 
     // And then redeem it on nodeB.
     await b.halo.recover(seedPhrase);
-    expect(a.halo.isInitialized).toBeTruthy();
-    expect(b.halo.isInitialized).toBeTruthy();
+    expect(a.halo.identity).toBeTruthy();
+    expect(b.halo.identity).toBeTruthy();
 
     // Now create a Party on A and make sure it gets opened on both A and B.
     expect(a.queryParties().value.length).toBe(0);

package/src/echo.ts

@@ -376,7 +376,7 @@ export class ECHO {
     assert(this._partyManager.isOpen, new InvalidStateError());
 
     const actualSecretProvider =
-      secretProvider ?? OfflineInvitationClaimer.createSecretProvider(this.halo.identity ?? raise(new IdentityNotInitializedError()));
+      secretProvider ?? OfflineInvitationClaimer.createSecretProvider(this.halo.identity?.createCredentialsSigner() ?? raise(new IdentityNotInitializedError()));
 
     return this._partyManager.joinParty(invitationDescriptor, actualSecretProvider);
   }

package/src/halo/halo-party.ts

@@ -163,8 +163,7 @@ export class HaloParty {
       this._networkManager,
       this._feedProvider,
       peerId,
-      createCredentialsProvider(this._credentialsSigner, this._partyCore.key, writeFeed.key),
-      this._partyCore.processor.getActiveFeedSet()
+      createCredentialsProvider(this._credentialsSigner, this._partyCore.key, writeFeed.key)
     );
 
     // Replication.

package/src/halo/halo.ts

@@ -46,6 +46,8 @@ export class HALO {
   private readonly _keyring: Keyring;
   private readonly _identityManager: IdentityManager;
 
+  private _isOpen = false;
+
   constructor ({
     keyring,
     networkManager,
@@ -85,7 +87,7 @@ export class HALO {
    * Whether the current identity manager has been initialized.
    */
   get isInitialized (): boolean {
-    return this._identityManager.initialized;
+    return this._isOpen;
   }
 
   /**
@@ -129,6 +131,7 @@ export class HALO {
    *
    * Loads the saved identity from disk. Is called by client.
    */
+  @synchronized
   async open (onProgressCallback?: ((progress: OpenProgress) => void) | undefined) {
     // TODO(burdon): Replace with events.
     onProgressCallback?.({ haloOpened: false });
@@ -137,12 +140,16 @@ export class HALO {
     await this._identityManager.loadFromStorage();
 
     onProgressCallback?.({ haloOpened: true });
+
+    this._isOpen = true;
   }
 
   /**
    * Closes HALO. Automatically called when client is destroyed.
    */
+  @synchronized
   async close () {
+    this._isOpen = false;
     await this._identityManager.close();
   }
 

package/src/halo/identity-manager.ts

@@ -7,6 +7,7 @@ import debug from 'debug';
 
 import { Event, synchronized, waitForCondition } from '@dxos/async';
 import { Filter, KeyRecord, Keyring, KeyType, SecretProvider } from '@dxos/credentials';
+import { failUndefined } from '@dxos/debug';
 
 import { InvitationDescriptor } from '../invitations';
 import { MetadataStore } from '../pipeline';
@@ -34,25 +35,23 @@ export class IdentityManager {
     return this._identity;
   }
 
-  get initialized (): boolean {
-    return this._identity !== undefined &&
-      !!this._identity.halo &&
-      this._identity.halo.isOpen &&
-      !!this._identity.halo!.memberKeys.length &&
-      !!this._identity.halo!.identityGenesis &&
-      !!this._identity.deviceKeyChain;
-  }
-
   private async _initialize (halo: HaloParty) {
     assert(halo.isOpen, 'HALO must be open.');
 
-    this._identity = new Identity(this._keyring, halo);
-
-    // Wait for the minimum set of keys and messages we need for proper function.
-    await waitForCondition(() => this.initialized);
+    // Wait for the minimum set of keys and messages we need for proper function:
+    //
+    // - KeyAdmit message for the current device so we can build the device KeyChain.
+    // - Identity genesis so it can be copied into newly joined parties.
+    //
+    const deviceKey = this._keyring.findKey(Keyring.signingFilter({ type: KeyType.DEVICE })) ?? failUndefined();
+    await waitForCondition(() =>
+      halo.processor.isMemberKey(deviceKey.publicKey) &&
+      halo.identityGenesis
+    );
 
-    log('HALO initialized.');
+    this._identity = new Identity(this._keyring, halo);
     this.ready.emit();
+    log('HALO initialized.');
   }
 
   async close () {

package/src/halo/identity.ts

@@ -2,13 +2,14 @@
 // Copyright 2021 DXOS.org
 //
 
+import assert from 'assert';
 import debug from 'debug';
 
-import { Filter, KeyChain, KeyRecord, Keyring, KeyType, Signer } from '@dxos/credentials';
-import { raise } from '@dxos/debug';
+import { Filter, KeyChain, KeyRecord, Keyring, KeyType, SignedMessage, Signer } from '@dxos/credentials';
+import { failUndefined } from '@dxos/debug';
 
-import { IdentityNotInitializedError } from '../errors';
 import { CredentialsSigner } from '../protocol/credentials-signer';
+import { IdentityCredentials } from '../protocol/identity-credentials';
 import { ContactManager } from './contact-manager';
 import { HaloParty } from './halo-party';
 import { Preferences } from './preferences';
@@ -20,10 +21,10 @@ const log = debug('dxos:echo-db:identity');
  *
  * Acts as a read-only view into IdentityManager.
  */
-export class Identity {
-  private _identityKey?: KeyRecord;
-  private _deviceKey?: KeyRecord;
-  private _deviceKeyChain?: KeyChain;
+export class Identity implements IdentityCredentials {
+  private readonly _identityKey: KeyRecord;
+  private readonly _deviceKey: KeyRecord;
+  private readonly _deviceKeyChain: KeyChain;
 
   /**
    * @param _halo HALO party. Must be open.
@@ -31,54 +32,55 @@ export class Identity {
   constructor (
     private readonly _keyring: Keyring,
     private readonly _halo: HaloParty
-  ) {}
+  ) {
+    this._identityKey = this._keyring.findKey(Filter.matches({ type: KeyType.IDENTITY, own: true, trusted: true })) ?? failUndefined();
+    this._deviceKey = this._keyring.findKey(Keyring.signingFilter({ type: KeyType.DEVICE })) ?? failUndefined();
+    this._deviceKeyChain = getDeviceKeyChainFromHalo(this._halo, this.deviceKey);
+    assert(this._halo.identityGenesis);
+  }
 
   get signer (): Signer {
     return this._keyring;
   }
 
-  get identityKey (): KeyRecord | undefined {
-    if (!this._identityKey) {
-      this._identityKey = this._keyring.findKey(Filter.matches({ type: KeyType.IDENTITY, own: true, trusted: true }));
-    }
+  get keyring (): Keyring {
+    return this._keyring;
+  }
 
+  get identityKey (): KeyRecord {
     return this._identityKey;
   }
 
-  get deviceKey (): KeyRecord | undefined {
-    if (!this._deviceKey) {
-      this._deviceKey = this._keyring.findKey(Keyring.signingFilter({ type: KeyType.DEVICE }));
-    }
-
+  get deviceKey (): KeyRecord {
     return this._deviceKey;
   }
 
-  get deviceKeyChain (): KeyChain | undefined {
-    if (!this._deviceKeyChain) {
-      this._deviceKeyChain = this.deviceKey && this._halo ? getDeviceKeyChainFromHalo(this._halo, this.deviceKey) : undefined;
-    }
-
+  get deviceKeyChain (): KeyChain {
     return this._deviceKeyChain;
   }
 
-  get preferences (): Preferences | undefined {
-    return this._halo?.preferences;
+  get displayName (): string | undefined {
+    return this.identityInfo?.signed.payload.displayName;
   }
 
-  get contacts (): ContactManager | undefined {
-    return this._halo?.contacts;
+  /**
+   * Contains profile username.
+   * Can be missing if the username wasn't provided when profile was created.
+   */
+  get identityInfo (): SignedMessage | undefined {
+    return this._halo.identityInfo;
   }
 
-  get displayName (): string | undefined {
-    return this.identityInfo?.signed.payload.displayName;
+  get identityGenesis (): SignedMessage {
+    return this._halo.identityGenesis ?? failUndefined();
   }
 
-  get identityInfo () {
-    return this._halo?.identityInfo;
+  get preferences (): Preferences {
+    return this._halo.preferences;
   }
 
-  get identityGenesis () {
-    return this._halo?.identityGenesis;
+  get contacts (): ContactManager {
+    return this._halo.contacts;
   }
 
   /**
@@ -88,16 +90,12 @@ export class Identity {
     return this._halo;
   }
 
-  get keyring () {
-    return this._keyring;
-  }
-
   createCredentialsSigner (): CredentialsSigner {
     return new CredentialsSigner(
       this._keyring,
-      () => this.identityKey ?? raise(new IdentityNotInitializedError()),
-      () => this.deviceKey ?? raise(new IdentityNotInitializedError()),
-      () => this.deviceKeyChain ?? this.deviceKey ?? raise(new IdentityNotInitializedError())
+      this.identityKey,
+      this.deviceKey,
+      this.deviceKeyChain
     );
   }
 }
@@ -112,7 +110,7 @@ function getDeviceKeyChainFromHalo (halo: HaloParty, deviceKey: KeyRecord) {
       halo.feedKeys
     );
   } catch (err: any) {
-    log('Unable to locate device KeyChain:', err); // TODO(burdon): ???
-    return undefined;
+    log('Unable to locate device KeyChain:', err);
+    throw err;
   }
 }

package/src/invitations/offline-invitation-claimer.ts

@@ -21,11 +21,10 @@ import {
   codec
 } from '@dxos/credentials';
 import { keyToBuffer, keyToString, PublicKey, randomBytes } from '@dxos/crypto';
-import { raise } from '@dxos/debug';
 import { FullyConnectedTopology, NetworkManager } from '@dxos/network-manager';
 
-import { IdentityNotInitializedError, InvalidInvitationError } from '../errors';
-import { Identity } from '../halo';
+import { InvalidInvitationError } from '../errors';
+import { CredentialsSigner } from '../protocol';
 import { greetingProtocolProvider } from './greeting-protocol-provider';
 import { GreetingState } from './greeting-responder';
 import { InvitationDescriptor, InvitationDescriptorType } from './invitation-descriptor';
@@ -170,17 +169,17 @@ export class OfflineInvitationClaimer {
   }
 
   // The secretProvider should provide an `Auth` message signed directly by the Identity key.
-  static createSecretProvider (identity: Identity): SecretProvider {
+  static createSecretProvider (credentials: CredentialsSigner): SecretProvider {
     return async (info?: SecretInfo) => {
       return Buffer.from(codec.encode(
         /* The signed portion of the Auth message includes the ID and authNonce provided
          * by the `info` object. These values will be validated on the other end.
          */
         createAuthMessage(
-          identity.signer,
+          credentials.signer,
           info!.id.value,
-          identity.identityKey ?? raise(new IdentityNotInitializedError()),
-          identity.deviceKeyChain ?? raise(new IdentityNotInitializedError()),
+          credentials.getIdentityKey(),
+          credentials.getDeviceSigningKeys(),
           undefined,
           info!.authNonce.value)
       ));

package/src/parties/data-party.test.ts

@@ -0,0 +1,212 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import expect from 'expect';
+import { it as test } from 'mocha';
+
+import { createKeyAdmitMessage, createPartyGenesisMessage, defaultSecretProvider, KeyHint, Keyring, KeyType, codec as haloCodec } from '@dxos/credentials';
+import { PublicKey } from '@dxos/crypto';
+import { codec } from '@dxos/echo-protocol';
+import { FeedStore } from '@dxos/feed-store';
+import { ModelFactory } from '@dxos/model-factory';
+import { NetworkManager } from '@dxos/network-manager';
+import { ObjectModel } from '@dxos/object-model';
+import { createStorage, StorageType } from '@dxos/random-access-multi-storage';
+
+import { createDataPartyAdmissionMessages, defaultInvitationAuthenticator, GreetingInitiator } from '../invitations';
+import { MetadataStore, PartyFeedProvider } from '../pipeline';
+import { createAuthenticator, createCredentialsProvider } from '../protocol';
+import { createTestIdentityCredentials, deriveTestDeviceCredentials, IdentityCredentials } from '../protocol/identity-credentials';
+import { SnapshotStore } from '../snapshots';
+import { createRamStorage } from '../util';
+import { DataParty } from './data-party';
+
+describe('DataParty', () => {
+  const createParty = async (identity: IdentityCredentials, partyKey: PublicKey, hints: KeyHint[]) => {
+    const metadataStore = new MetadataStore(createRamStorage());
+    const feedStore = new FeedStore(createStorage('', StorageType.RAM), { valueEncoding: codec });
+    const snapshotStore = new SnapshotStore(createStorage('', StorageType.RAM));
+    const modelFactory = new ModelFactory().registerModel(ObjectModel);
+    const networkManager = new NetworkManager();
+    const partyFeedProvider = new PartyFeedProvider(metadataStore, identity.keyring, feedStore, partyKey);
+
+    return new DataParty(
+      partyKey,
+      modelFactory,
+      snapshotStore,
+      partyFeedProvider,
+      identity.createCredentialsSigner(),
+      identity.preferences,
+      networkManager,
+      hints
+    );
+  };
+
+  test('open & close', async () => {
+    const keyring = new Keyring();
+    const identity = await createTestIdentityCredentials(keyring);
+    const partyKey = await keyring.createKeyRecord({ type: KeyType.PARTY });
+    const party = await createParty(identity, partyKey.publicKey, []);
+
+    await party.open();
+    await party.close();
+  });
+
+  test('edit data', async () => {
+    const keyring = new Keyring();
+    const identity = await createTestIdentityCredentials(keyring);
+    const partyKey = await keyring.createKeyRecord({ type: KeyType.PARTY });
+    const party = await createParty(identity, partyKey.publicKey, []);
+    await party.open();
+
+    const feed = await party.feedProvider.createOrOpenWritableFeed();
+    await party.processor.writeHaloMessage(createPartyGenesisMessage(
+      keyring,
+      partyKey,
+      feed.key,
+      partyKey
+    ));
+
+    await party.database.createItem({ type: 'test:item' });
+
+    await party.close();
+  });
+
+  test('authenticates its own credentials', async () => {
+    const keyring = new Keyring();
+    const identity = await createTestIdentityCredentials(keyring);
+    const partyKey = await keyring.createKeyRecord({ type: KeyType.PARTY });
+
+    const party = await createParty(identity, partyKey.publicKey, []);
+    await party.open();
+    const feed = await party.feedProvider.createOrOpenWritableFeed();
+    await party.processor.writeHaloMessage(createPartyGenesisMessage(
+      keyring,
+      partyKey,
+      feed.key,
+      partyKey
+    ));
+
+    const authenticator = createAuthenticator(party.processor, identity.createCredentialsSigner());
+    const credentialsProvider = createCredentialsProvider(identity.createCredentialsSigner(), party.key, feed.key);
+
+    const wrappedCredentials = haloCodec.decode(credentialsProvider.get());
+    expect(await authenticator.authenticate(wrappedCredentials.payload)).toEqual(true);
+
+    await party.close();
+  });
+
+  test('authenticates another device', async () => {
+    const keyring = new Keyring();
+    const identityA = await createTestIdentityCredentials(keyring);
+    const partyKey = await keyring.createKeyRecord({ type: KeyType.PARTY });
+
+    const party = await createParty(identityA, partyKey.publicKey, []);
+    await party.open();
+    const feed = await party.feedProvider.createOrOpenWritableFeed();
+    await party.processor.writeHaloMessage(createPartyGenesisMessage(
+      keyring,
+      partyKey,
+      feed.key,
+      partyKey
+    ));
+    const authenticator = createAuthenticator(party.processor, identityA.createCredentialsSigner());
+
+    const identityB = await deriveTestDeviceCredentials(identityA);
+    const credentialsProvider = createCredentialsProvider(identityB.createCredentialsSigner(), party.key, feed.key);
+
+    const wrappedCredentials = haloCodec.decode(credentialsProvider.get());
+    expect(await authenticator.authenticate(wrappedCredentials.payload)).toEqual(true);
+
+    await party.close();
+  });
+
+  test('two instances replicating', async () => {
+    const keyring = new Keyring();
+    const identityA = await createTestIdentityCredentials(keyring);
+    const partyKey = await keyring.createKeyRecord({ type: KeyType.PARTY });
+
+    const partyA = await createParty(identityA, partyKey.publicKey, []);
+    await partyA.open();
+    const feedA = await partyA.feedProvider.createOrOpenWritableFeed();
+    await partyA.processor.writeHaloMessage(createPartyGenesisMessage(
+      keyring,
+      partyKey,
+      feedA.key,
+      partyKey
+    ));
+    await partyA.processor.writeHaloMessage(createKeyAdmitMessage(
+      keyring,
+      partyKey.publicKey,
+      identityA.identityKey,
+      [partyKey]
+    ));
+
+    const identityB = await deriveTestDeviceCredentials(identityA);
+    const partyB = await createParty(identityB, partyKey.publicKey, [
+      { type: KeyType.FEED, publicKey: feedA.key }
+    ]);
+    await partyB.open();
+
+    await partyA.database.createItem({ type: 'test:item-a' });
+    await partyB.database.waitForItem({ type: 'test:item-a' });
+
+    await partyB.database.createItem({ type: 'test:item-b' });
+    await partyA.database.waitForItem({ type: 'test:item-b' });
+
+    await partyA.close();
+    await partyB.close();
+  });
+
+  test('invitations', async () => {
+    const identityA = await createTestIdentityCredentials(new Keyring());
+    const partyKeyA = await identityA.keyring.createKeyRecord({ type: KeyType.PARTY });
+
+    const partyA = await createParty(identityA, partyKeyA.publicKey, []);
+    await partyA.open();
+    const feedA = await partyA.feedProvider.createOrOpenWritableFeed();
+    await partyA.processor.writeHaloMessage(createPartyGenesisMessage(
+      identityA.keyring,
+      partyKeyA,
+      feedA.key,
+      partyKeyA
+    ));
+    await partyA.processor.writeHaloMessage(createKeyAdmitMessage(
+      identityA.keyring,
+      partyKeyA.publicKey,
+      identityA.identityKey,
+      [partyKeyA]
+    ));
+
+    const invitation = await partyA.invitationManager.createInvitation(defaultInvitationAuthenticator);
+
+    const identityB = await createTestIdentityCredentials(new Keyring());
+    const initiator = new GreetingInitiator(
+      new NetworkManager(),
+      invitation,
+      async (partyKey, nonce) => [createDataPartyAdmissionMessages(
+        identityB.createCredentialsSigner(),
+        partyKey,
+        identityB.identityGenesis,
+        nonce
+      )]
+    );
+
+    await initiator.connect();
+    const { partyKey: partyKeyB, hints: hintsB } = await initiator.redeemInvitation(defaultSecretProvider);
+    expect(partyKeyB.equals(partyKeyA.publicKey));
+    const partyB = await createParty(identityB, partyKeyB, hintsB);
+    await partyB.open();
+    await initiator.destroy();
+
+    await partyA.database.createItem({ type: 'test:item-a' });
+    await partyB.database.waitForItem({ type: 'test:item-a' });
+
+    await partyB.database.createItem({ type: 'test:item-b' });
+    await partyA.database.waitForItem({ type: 'test:item-b' });
+
+    await partyA.close();
+    await partyB.close();
+  });
+});

package/src/parties/data-party.ts

@@ -175,8 +175,7 @@ export class DataParty {
       this._networkManager,
       this._feedProvider,
       deviceKey.publicKey,
-      createCredentialsProvider(this._credentialsSigner, this._partyCore.key, writeFeed.key),
-      this._partyCore.processor.getActiveFeedSet()
+      createCredentialsProvider(this._credentialsSigner, this._partyCore.key, writeFeed.key)
     );
 
     await this._protocol.start([