@dxos/echo-db 2.33.1-dev.83d113fe → 2.33.1-dev.eb88db79

package/src/echo.ts

@@ -6,6 +6,7 @@ import assert from 'assert';
 import debug from 'debug';
 import memdown from 'memdown';
 
+import { synchronized } from '@dxos/async';
 import { Keyring, KeyStore, SecretProvider } from '@dxos/credentials';
 import { PublicKey } from '@dxos/crypto';
 import { InvalidStateError } from '@dxos/debug';
@@ -237,6 +238,7 @@ export class ECHO {
    *
    * Previously active parties will be opened and will begin replication.
    */
+  @synchronized
   async open (onProgressCallback?: ((progress: OpenProgress) => void) | undefined) {
     if (this.isOpen) {
       return;
@@ -258,6 +260,7 @@ export class ECHO {
   /**
    * Closes the ECHO instance.
    */
+   @synchronized
   async close () {
     if (!this.isOpen) {
       return;
@@ -276,107 +279,107 @@ export class ECHO {
     await this.networkManager.destroy();
   }
 
-  /**
+   /**
    * Removes all data and closes this ECHO instance.
    *
    * The instance will be in an unusable state at this point and a page refresh is recommended.
    */
-  // TODO(burdon): Enable re-open.
-  async reset () {
-    await this.close();
-
-    try {
-      if (this._feedStore.storage.destroy) {
-        await this._feedStore.storage.destroy();
-      }
-    } catch (err: any) {
-      error('Error clearing feed storage:', err);
-    }
-
-    await this.halo.reset();
-
-    try {
-      await this._snapshotStore.clear();
-    } catch (err: any) {
-      error('Error clearing snapshot storage:', err);
-    }
-
-    try {
-      await this._metadataStore.clear();
-    } catch (err: any) {
-      error('Error clearing metadata storage:', err);
-    }
-  }
-
-  //
-  // Parties.
-  //
-
-  /**
+   // TODO(burdon): Enable re-open.
+   async reset () {
+     await this.close();
+
+     try {
+       if (this._feedStore.storage.destroy) {
+         await this._feedStore.storage.destroy();
+       }
+     } catch (err: any) {
+       error('Error clearing feed storage:', err);
+     }
+
+     await this.halo.reset();
+
+     try {
+       await this._snapshotStore.clear();
+     } catch (err: any) {
+       error('Error clearing snapshot storage:', err);
+     }
+
+     try {
+       await this._metadataStore.clear();
+     } catch (err: any) {
+       error('Error clearing metadata storage:', err);
+     }
+   }
+
+   //
+   // Parties.
+   //
+
+   /**
    * Creates a new party.
    */
-  async createParty (): Promise<PartyInternal> {
-    await this.open();
+   async createParty (): Promise<PartyInternal> {
+     await this.open();
 
-    const party = await this._partyManager.createParty();
-    await party.open();
+     const party = await this._partyManager.createParty();
+     await party.open();
 
-    return party;
-  }
+     return party;
+   }
 
-  /**
+   /**
    * Clones an existing party from a snapshot.
    * @param snapshot
    */
-  async cloneParty (snapshot: PartySnapshot) {
-    await this.open();
+   async cloneParty (snapshot: PartySnapshot) {
+     await this.open();
 
-    const party = await this._partyManager.cloneParty(snapshot);
-    await party.open();
+     const party = await this._partyManager.cloneParty(snapshot);
+     await party.open();
 
-    return party;
-  }
+     return party;
+   }
 
-  /**
+   /**
    * Returns an individual party by it's key.
    * @param {PartyKey} partyKey
    */
-  getParty (partyKey: PartyKey): PartyInternal | undefined {
-    if (!this._partyManager.isOpen) {
-      throw new InvalidStateError();
-    }
+   getParty (partyKey: PartyKey): PartyInternal | undefined {
+     if (!this._partyManager.isOpen) {
+       throw new InvalidStateError();
+     }
 
-    const party = this._partyManager.parties.find(party => party.key.equals(partyKey));
-    return party;
-  }
+     const party = this._partyManager.parties.find(party => party.key.equals(partyKey));
+     return party;
+   }
 
-  /**
+   /**
    * Queries for a set of Parties matching the optional filter.
    * @param {PartyFilter} filter
    */
-  // eslint-disable-next-line unused-imports/no-unused-vars
-  queryParties (filter?: PartyFilter): ResultSet<PartyInternal> {
-    if (!this._partyManager.isOpen) {
-      throw new InvalidStateError();
-    }
-
-    return new ResultSet(
-      this._partyManager.update.discardParameter(),
-      () => this._partyManager.parties
-    );
-  }
-
-  /**
+   // eslint-disable-next-line unused-imports/no-unused-vars
+   queryParties (filter?: PartyFilter): ResultSet<PartyInternal> {
+     if (!this._partyManager.isOpen) {
+       throw new InvalidStateError();
+     }
+
+     return new ResultSet(
+       this._partyManager.update.discardParameter(),
+       () => this._partyManager.parties
+     );
+   }
+
+   /**
    * Joins a party that was created by another peer and starts replicating with it.
    * @param invitationDescriptor Invitation descriptor passed from another peer.
    * @param secretProvider Shared secret provider, the other peer creating the invitation must have the same secret.
    */
-  async joinParty (invitationDescriptor: InvitationDescriptor, secretProvider?: SecretProvider): Promise<PartyInternal> {
-    assert(this._partyManager.isOpen, new InvalidStateError());
+   async joinParty (invitationDescriptor: InvitationDescriptor, secretProvider?: SecretProvider): Promise<PartyInternal> {
+     assert(this._partyManager.isOpen, new InvalidStateError());
 
-    const actualSecretProvider =
+     const actualSecretProvider =
       secretProvider ?? OfflineInvitationClaimer.createSecretProvider(this.halo.identity);
 
-    return this._partyManager.joinParty(invitationDescriptor, actualSecretProvider);
-  }
+     return this._partyManager.joinParty(invitationDescriptor, actualSecretProvider);
+   }
 }

package/src/invitations/greeting-initiator.ts

@@ -8,7 +8,6 @@ import debug from 'debug';
 import { waitForEvent } from '@dxos/async';
 import {
   createEnvelopeMessage,
-  createFeedAdmitMessage,
   createGreetingBeginMessage,
   createGreetingFinishMessage,
   createGreetingHandshakeMessage,
@@ -23,7 +22,6 @@ import {
   ERR_GREET_CONNECTED_TO_SWARM_TIMEOUT
 } from '@dxos/credentials';
 import { keyToString, PublicKey } from '@dxos/crypto';
-import { PartyKey } from '@dxos/echo-protocol';
 import { FullyConnectedTopology, NetworkManager } from '@dxos/network-manager';
 
 import { Identity } from '../halo';
@@ -54,8 +52,7 @@ export class GreetingInitiator {
   constructor (
     private readonly _networkManager: NetworkManager,
     private readonly _identity: Identity,
-    private readonly _invitationDescriptor: InvitationDescriptor,
-    private readonly _feedInitializer: (partyKey: PartyKey) => Promise<PublicKey>
+    private readonly _invitationDescriptor: InvitationDescriptor
   ) {
     assert(InvitationDescriptorType.INTERACTIVE === this._invitationDescriptor.type);
   }
@@ -149,8 +146,6 @@ export class GreetingInitiator {
     const { nonce } = handshakeResponse;
     const partyKey = handshakeResponse.partyKey;
 
-    const feedKey = await this._feedInitializer(partyKey);
-
     const credentialMessages = [];
     if (haloInvitation) {
       assert(this._identity.deviceKey, 'Device key required');
@@ -164,16 +159,6 @@ export class GreetingInitiator {
           [],
           nonce)
       );
-
-      // And Feed, signed for by the FEED and the DEVICE.
-      credentialMessages.push(
-        createFeedAdmitMessage(
-          this._identity.signer,
-          partyKey,
-          feedKey,
-          [this._identity.deviceKey],
-          nonce)
-      );
     } else {
       assert(this._identity.deviceKeyChain, 'Device key required');
       assert(this._identity.identityGenesis, 'Identity genesis message required');
@@ -187,16 +172,6 @@ export class GreetingInitiator {
           [this._identity.deviceKeyChain],
           nonce)
       );
-
-      // And the Feed, signed for by the FEED and by the DEVICE keychain, as above.
-      credentialMessages.push(
-        createFeedAdmitMessage(
-          this._identity.signer,
-          partyKey,
-          feedKey,
-          [this._identity.deviceKeyChain],
-          nonce)
-      );
     }
 
     // Send the signed payload to the greeting responder.

package/src/invitations/halo-recovery-initiator.ts

@@ -7,7 +7,6 @@ import debug from 'debug';
 
 import { waitForEvent } from '@dxos/async';
 import {
-  Authenticator,
   ClaimResponse,
   Keyring,
   KeyType,
@@ -17,7 +16,8 @@ import {
   createGreetingClaimMessage,
   SecretProvider,
   SecretValidator,
-  SignedMessage
+  SignedMessage,
+  codec
 } from '@dxos/credentials';
 import { keyToBuffer, keyToString, PublicKey, randomBytes, verify } from '@dxos/crypto';
 import { raise } from '@dxos/debug';
@@ -144,7 +144,7 @@ export class HaloRecoveryInitiator {
 
   // The secretProvider should provide an `Auth` message signed directly by the Identity key.
   createSecretProvider (): SecretProvider {
-    return async (info: any) => Buffer.from(Authenticator.encodePayload(
+    return async (info: any) => Buffer.from(codec.encode(
       /* The signed portion of the Auth message includes the ID and authNonce provided
        * by "info". These values will be validated on the other end.
        */
@@ -182,7 +182,7 @@ export class HaloRecoveryInitiator {
       });
 
       const secretValidator: SecretValidator = async (invitation, secret) => {
-        const { payload: authMessage } = Authenticator.decodePayload(secret);
+        const { payload: authMessage } = codec.decode(secret);
 
         return keyring.verify(<unknown>authMessage as SignedMessage) &&
           authMessage.signed.payload.partyKey.equals(invitation.id) &&

package/src/invitations/offline-invitation-claimer.ts

@@ -7,7 +7,6 @@ import debug from 'debug';
 
 import { waitForEvent } from '@dxos/async';
 import {
-  Authenticator,
   ClaimResponse,
   Keyring,
   KeyType,
@@ -18,7 +17,8 @@ import {
   SecretInfo,
   SecretProvider,
   SecretValidator,
-  SignedMessage
+  SignedMessage,
+  codec
 } from '@dxos/credentials';
 import { keyToBuffer, keyToString, PublicKey, randomBytes } from '@dxos/crypto';
 import { raise } from '@dxos/debug';
@@ -156,7 +156,7 @@ export class OfflineInvitationClaimer {
       });
 
       const secretValidator: SecretValidator = async (invitation, secret) => {
-        const { payload: authMessage } = Authenticator.decodePayload(secret);
+        const { payload: authMessage } = codec.decode(secret);
 
         return keyring.verify(<unknown>authMessage as SignedMessage) &&
           authMessage.signed.payload.partyKey.equals(invitation.id) &&
@@ -172,7 +172,7 @@ export class OfflineInvitationClaimer {
   // The secretProvider should provide an `Auth` message signed directly by the Identity key.
   static createSecretProvider (identity: Identity): SecretProvider {
     return async (info?: SecretInfo) => {
-      return Buffer.from(Authenticator.encodePayload(
+      return Buffer.from(codec.encode(
         /* The signed portion of the Auth message includes the ID and authNonce provided
          * by the `info` object. These values will be validated on the other end.
          */

package/src/parties/authenticator.ts

@@ -0,0 +1,31 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import debug from 'debug';
+
+import { Authenticator, createEnvelopeMessage, PartyAuthenticator } from '@dxos/credentials';
+
+import { IdentityProvider } from '../halo';
+import { PartyProcessor } from '../pipeline';
+
+const log = debug('dxos:echo-db:authenticator');
+
+export function createAuthenticator (partyProcessor: PartyProcessor, identityProvider: IdentityProvider): Authenticator {
+  return new PartyAuthenticator(partyProcessor.state, async auth => {
+    if (auth.feedAdmit && auth.feedKey && !partyProcessor.isFeedAdmitted(auth.feedKey)) {
+      const deviceKeyChain = identityProvider().deviceKeyChain ?? identityProvider().deviceKey;
+      if (!deviceKeyChain) {
+        log('Not device key chain available to admit new member feed');
+        return;
+      }
+
+      await partyProcessor.writeHaloMessage(createEnvelopeMessage(
+        identityProvider().keyring,
+        partyProcessor.partyKey,
+        auth.feedAdmit,
+        [deviceKeyChain]
+      ));
+    }
+  });
+}

package/src/parties/party-core.test.ts

@@ -5,24 +5,26 @@
 import expect from 'expect';
 import { it as test } from 'mocha';
 
+import { promiseTimeout } from '@dxos/async';
 import { createFeedAdmitMessage, createPartyGenesisMessage, Keyring, KeyType } from '@dxos/credentials';
-import { PublicKey } from '@dxos/crypto';
-import { codec } from '@dxos/echo-protocol';
+import { createId, PublicKey } from '@dxos/crypto';
+import { codec, Timeframe } from '@dxos/echo-protocol';
 import { FeedStore } from '@dxos/feed-store';
+import { createTestProtocolPair } from '@dxos/mesh-protocol';
 import { ModelFactory } from '@dxos/model-factory';
 import { ObjectModel } from '@dxos/object-model';
-import { createStorage, STORAGE_RAM } from '@dxos/random-access-multi-storage';
+import { createStorage, StorageType } from '@dxos/random-access-multi-storage';
 import { afterTest } from '@dxos/testutils';
 
 import { MetadataStore } from '../metadata';
-import { PartyFeedProvider } from '../pipeline';
+import { PartyFeedProvider, ReplicatorProtocolPluginFactory } from '../pipeline';
 import { SnapshotStore } from '../snapshots';
 import { createRamStorage } from '../util';
 import { PartyCore } from './party-core';
 
 describe('PartyCore', () => {
   const setup = async () => {
-    const storage = createStorage('', STORAGE_RAM);
+    const storage = createStorage('', StorageType.RAM);
     const feedStore = new FeedStore(storage, { valueEncoding: codec });
     afterTest(async () => feedStore.close());
 
@@ -31,7 +33,7 @@ describe('PartyCore', () => {
     const metadataStore = new MetadataStore(createRamStorage());
 
     const modelFactory = new ModelFactory().registerModel(ObjectModel);
-    const snapshotStore = new SnapshotStore(createStorage('', STORAGE_RAM));
+    const snapshotStore = new SnapshotStore(createStorage('', StorageType.RAM));
 
     const partyKey = await keyring.createKeyRecord({ type: KeyType.PARTY });
 
@@ -65,10 +67,7 @@ describe('PartyCore', () => {
       [partyKey]
     ));
 
-    // The Party key is an inception key; its SecretKey must be destroyed once the Party has been created.
-    await keyring.deleteSecretKey(partyKey);
-
-    return { party, feedKey: feed.key, feed, feedStore };
+    return { party, feedKey: feed.key, feed, feedStore, partyKey, keyring, partyFeedProvider };
   };
 
   test('create & have the feed key admitted', async () => {
@@ -115,4 +114,228 @@ describe('PartyCore', () => {
       expect(parent.children).toContain(child);
     }
   });
+
+  test('feed admit message triggers new feed to be opened', async () => {
+    const { party, partyKey, keyring, partyFeedProvider, feedStore } = await setup();
+
+    const feedKey = await keyring.createKeyRecord({ type: KeyType.FEED });
+
+    const eventFired = feedStore.feedOpenedEvent.waitForCount(1);
+    await party.processor.writeHaloMessage(createFeedAdmitMessage(
+      keyring,
+      party.key,
+      feedKey.publicKey,
+      [partyKey]
+    ));
+    await promiseTimeout(eventFired, 1000, new Error('timeout'));
+    expect(partyFeedProvider.getFeeds().find(k => k.key.equals(feedKey.publicKey))).toBeTruthy();
+  });
+
+  test('opens feed from hints', async () => {
+    const storage = createStorage('', StorageType.RAM);
+    const feedStore = new FeedStore(storage, { valueEncoding: codec });
+    afterTest(async () => feedStore.close());
+
+    const keyring = new Keyring();
+
+    const metadataStore = new MetadataStore(createRamStorage());
+
+    const modelFactory = new ModelFactory().registerModel(ObjectModel);
+    const snapshotStore = new SnapshotStore(createStorage('', StorageType.RAM));
+
+    const partyKey = await keyring.createKeyRecord({ type: KeyType.PARTY });
+
+    const partyFeedProvider = new PartyFeedProvider(metadataStore, keyring, feedStore, partyKey.publicKey);
+
+    const otherFeedKey = PublicKey.random();
+
+    const party = new PartyCore(
+      partyKey.publicKey,
+      partyFeedProvider,
+      modelFactory,
+      snapshotStore,
+      PublicKey.random()
+    );
+
+    await partyFeedProvider.createOrOpenWritableFeed();
+
+    const feedOpened = feedStore.feedOpenedEvent.waitForCount(1);
+
+    await party.open([{ type: KeyType.FEED, publicKey: otherFeedKey }]);
+    afterTest(async () => party.close());
+
+    await feedOpened;
+
+    expect(partyFeedProvider.getFeeds().some(k => k.key.equals(otherFeedKey))).toEqual(true);
+  });
+
+  test('manually create item', async () => {
+    const { party, partyFeedProvider } = await setup();
+    await party.open();
+
+    const feed = await partyFeedProvider.createOrOpenWritableFeed();
+
+    const itemId = createId();
+    await feed.feed.append({
+      echo: {
+        itemId,
+        genesis: {
+          itemType: 'dxos:example',
+          modelType: ObjectModel.meta.type
+        },
+        timeframe: new Timeframe()
+      }
+    });
+
+    await promiseTimeout(party.database.waitForItem({ id: itemId }), 1000, new Error('timeout'));
+  });
+
+  test('admit a second feed to the party', async () => {
+    const { party, keyring, partyKey, feedStore } = await setup();
+    await party.open();
+
+    const feedKey = await keyring.createKeyRecord({ type: KeyType.FEED });
+    const fullKey = keyring.getFullKey(feedKey.publicKey);
+    const feed2 = await feedStore.openReadWriteFeed(fullKey!.publicKey, fullKey!.secretKey!);
+
+    await party.processor.writeHaloMessage(createFeedAdmitMessage(
+      keyring,
+      party.key,
+      feed2.key,
+      [partyKey]
+    ));
+
+    const itemId = createId();
+    await feed2.append({
+      echo: {
+        itemId,
+        genesis: {
+          itemType: 'dxos:example',
+          modelType: ObjectModel.meta.type
+        },
+        timeframe: new Timeframe()
+      }
+    });
+
+    await promiseTimeout(party.database.waitForItem({ id: itemId }), 1000, new Error('timeout'));
+  });
+
+  test('admit feed and then open it', async () => {
+    const { party, keyring, partyKey, feedStore } = await setup();
+    await party.open();
+
+    const feedKey = await keyring.createKeyRecord({ type: KeyType.FEED });
+    const fullKey = keyring.getFullKey(feedKey.publicKey);
+
+    await party.processor.writeHaloMessage(createFeedAdmitMessage(
+      keyring,
+      party.key,
+      feedKey.publicKey,
+      [partyKey]
+    ));
+
+    const feed2 = await feedStore.openReadWriteFeed(fullKey!.publicKey, fullKey!.secretKey!);
+    const itemId = createId();
+    await feed2.append({
+      echo: {
+        itemId,
+        genesis: {
+          itemType: 'dxos:example',
+          modelType: ObjectModel.meta.type
+        },
+        timeframe: new Timeframe()
+      }
+    });
+
+    await promiseTimeout(party.database.waitForItem({ id: itemId }), 1000, new Error('timeout'));
+  });
+
+  test('self-admitting feed with a hint', async () => {
+    const { party, keyring, partyKey, feedStore } = await setup();
+    await party.open();
+
+    const feedKey = await keyring.createKeyRecord({ type: KeyType.FEED });
+    const fullKey = keyring.getFullKey(feedKey.publicKey);
+    const feed2 = await feedStore.openReadWriteFeed(fullKey!.publicKey, fullKey!.secretKey!);
+
+    await party.processor.takeHints([{
+      type: KeyType.FEED,
+      publicKey: feedKey.publicKey
+    }]);
+
+    await feed2.append({
+      halo: createFeedAdmitMessage(
+        keyring,
+        party.key,
+        feedKey.publicKey,
+        [partyKey]
+      )
+    });
+
+    const itemId = createId();
+    await feed2.append({
+      echo: {
+        itemId,
+        genesis: {
+          itemType: 'dxos:example',
+          modelType: ObjectModel.meta.type
+        },
+        timeframe: new Timeframe()
+      }
+    });
+
+    await promiseTimeout(party.database.waitForItem({ id: itemId }), 1000, new Error('timeout'));
+  });
+
+  test('two instances replicating', async () => {
+    const peer1 = await setup();
+
+    const storage = createStorage('', StorageType.RAM);
+    const feedStore = new FeedStore(storage, { valueEncoding: codec });
+    afterTest(async () => feedStore.close());
+
+    const metadataStore = new MetadataStore(createRamStorage());
+
+    const modelFactory = new ModelFactory().registerModel(ObjectModel);
+    const snapshotStore = new SnapshotStore(createStorage('', StorageType.RAM));
+
+    const partyFeedProvider = new PartyFeedProvider(metadataStore, peer1.keyring, feedStore, peer1.party.key);
+
+    const party2 = new PartyCore(
+      peer1.party.key,
+      partyFeedProvider,
+      modelFactory,
+      snapshotStore,
+      PublicKey.random()
+    );
+
+    const feed2 = await partyFeedProvider.createOrOpenWritableFeed();
+
+    await peer1.party.processor.writeHaloMessage(createFeedAdmitMessage(
+      peer1.keyring,
+      peer1.party.key,
+      feed2.key,
+      [peer1.partyKey]
+    ));
+
+    await party2.open([{
+      publicKey: peer1.feedKey,
+      type: KeyType.FEED
+    }]);
+    afterTest(async () => party2.close());
+
+    createTestProtocolPair(
+      new ReplicatorProtocolPluginFactory(
+        peer1.partyFeedProvider,
+        peer1.party.processor.getActiveFeedSet()
+      ).createPlugins().map(r => r.createExtension()),
+      new ReplicatorProtocolPluginFactory(
+        partyFeedProvider,
+        peer1.party.processor.getActiveFeedSet()
+      ).createPlugins().map(r => r.createExtension())
+    );
+
+    const item1 = await peer1.party.database.createItem();
+    await promiseTimeout(party2.database.waitForItem({ id: item1.id }), 1000, new Error('timeout'));
+  });
 });

package/src/parties/party-core.ts

@@ -113,9 +113,14 @@ export class PartyCore {
 
     if (!this._partyProcessor) {
       this._partyProcessor = new PartyProcessor(this._partyKey);
-      if (keyHints.length > 0) {
-        await this._partyProcessor.takeHints(keyHints);
-      }
+    }
+    // Automatically open new admitted feeds.
+    this._subscriptions.push(this._partyProcessor.feedAdded.on(feed => {
+      void this._feedProvider.createOrOpenReadOnlyFeed(feed);
+    }));
+
+    if (keyHints.length > 0) {
+      await this._partyProcessor.takeHints(keyHints);
     }
 
     //

package/src/parties/party-factory.ts

@@ -205,12 +205,7 @@ export class PartyFactory {
     const initiator = new GreetingInitiator(
       this._networkManager,
       identity,
-      invitationDescriptor,
-      async partyKey => {
-        const feedProvider = this._createFeedProvider(partyKey);
-        const feed = await feedProvider.createOrOpenWritableFeed();
-        return feed.key;
-      }
+      invitationDescriptor
     );
 
     await initiator.connect();