@dxos/echo-db 2.32.1-dev.ebff6d2e → 2.33.1-dev.0a5ae2e2

package/src/echo.test.ts

@@ -7,7 +7,7 @@ import debug from 'debug';
 import expect from 'expect';
 import { it as test } from 'mocha';
 
-import { latch, waitForCondition } from '@dxos/async';
+import { latch, promiseTimeout, waitForCondition } from '@dxos/async';
 import { defaultSecretProvider, defaultSecretValidator } from '@dxos/credentials';
 import { generateSeedPhrase, keyPairFromSeedPhrase, createKeyPair } from '@dxos/crypto';
 import { ObjectModel } from '@dxos/object-model';
@@ -329,6 +329,105 @@ describe('ECHO', () => {
     expect(a.queryParties().value[1].key).toEqual(b.queryParties().value[1].key);
   }).timeout(10_000);
 
+  test('Mutations from another device', async () => {
+    const a = await setup({ createProfile: true });
+    const b = await setup();
+
+    await a.createParty();
+
+    const invitation = await a.halo.createInvitation(defaultInvitationAuthenticator);
+    await b.halo.join(invitation, defaultSecretProvider);
+
+    // Check the initial party is opened.
+    await waitForCondition(() => b.queryParties().value.length === 1, 1000);
+
+    const partyA = a.queryParties().value[0];
+    await partyA.open();
+    const partyB = b.queryParties().value[0];
+    await partyB.open();
+
+    {
+      // Subscribe to Item updates on B.
+      const selection = partyA.database.select({ type: 'example:item/test' }).exec();
+      const updated = selection.update.waitFor(result => result.entities.length > 0);
+
+      // Create a new Item on A.
+      const itemB = await partyB.database
+        .createItem({ model: ObjectModel, type: 'example:item/test' }) as Item<any>;
+      log(`A created ${itemB.id}`);
+
+      // Now wait to see it on B.
+      await updated;
+      log(`B has ${itemB.id}`);
+
+      expect(selection.entities[0].id).toEqual(itemB.id);
+    }
+
+    await a.close();
+    await b.close();
+
+    await a.open();
+
+    {
+      const partyA = a.queryParties().first;
+
+      expect(partyA.database.select({ type: 'example:item/test' }).exec().entities.length > 0).toEqual(true);
+    }
+
+  }).timeout(10_000);
+
+  test.skip('3 devices', async () => {
+    const a = await setup({ createProfile: true });
+    const b = await setup();
+
+    await a.createParty();
+
+    await b.halo.join(await a.halo.createInvitation(defaultInvitationAuthenticator), defaultSecretProvider);
+
+    // Check the initial party is opened.
+    await waitForCondition(() => b.queryParties().value.length === 1, 1000);
+
+    const partyA = a.queryParties().value[0];
+    await partyA.open();
+    const partyB = b.queryParties().value[0];
+    await partyB.open();
+
+    {
+      // Subscribe to Item updates on B.
+      const selection = partyA.database.select({ type: 'example:item/test' }).exec();
+      const updated = selection.update.waitFor(result => result.entities.length > 0);
+
+      // Create a new Item on A.
+      const itemB = await partyB.database
+        .createItem({ model: ObjectModel, type: 'example:item/test' }) as Item<any>;
+      log(`A created ${itemB.id}`);
+
+      // Now wait to see it on B.
+      await updated;
+      log(`B has ${itemB.id}`);
+
+      expect(selection.entities[0].id).toEqual(itemB.id);
+    }
+
+    await a.close();
+    await b.close();
+
+    await a.open();
+
+    {
+      const partyA = a.queryParties().first;
+
+      expect(partyA.database.select({ type: 'example:item/test' }).exec().entities.length > 0).toEqual(true);
+    }
+
+    const c = await setup();
+    await c.halo.join(await a.halo.createInvitation(defaultInvitationAuthenticator), defaultSecretProvider);
+    await waitForCondition(() => c.queryParties().value.length === 1, 1000);
+    const partyC = c.queryParties().first;
+
+    await promiseTimeout(partyC.database.waitForItem({ type: 'example:item/test' }), 1000, new Error('timeout'));
+  }).timeout(10_000);
+
   test('Two users, two devices each', async () => {
     const a1 = await setup({ createProfile: true });
     const a2 = await setup();

package/src/invitations/greeting-initiator.ts

@@ -8,7 +8,6 @@ import debug from 'debug';
 import { waitForEvent } from '@dxos/async';
 import {
   createEnvelopeMessage,
-  createFeedAdmitMessage,
   createGreetingBeginMessage,
   createGreetingFinishMessage,
   createGreetingHandshakeMessage,
@@ -23,7 +22,6 @@ import {
   ERR_GREET_CONNECTED_TO_SWARM_TIMEOUT
 } from '@dxos/credentials';
 import { keyToString, PublicKey } from '@dxos/crypto';
-import { PartyKey } from '@dxos/echo-protocol';
 import { FullyConnectedTopology, NetworkManager } from '@dxos/network-manager';
 
 import { Identity } from '../halo';
@@ -54,8 +52,7 @@ export class GreetingInitiator {
   constructor (
     private readonly _networkManager: NetworkManager,
     private readonly _identity: Identity,
-    private readonly _invitationDescriptor: InvitationDescriptor,
-    private readonly _feedInitializer: (partyKey: PartyKey) => Promise<PublicKey>
+    private readonly _invitationDescriptor: InvitationDescriptor
   ) {
     assert(InvitationDescriptorType.INTERACTIVE === this._invitationDescriptor.type);
   }
@@ -149,8 +146,6 @@ export class GreetingInitiator {
     const { nonce } = handshakeResponse;
     const partyKey = handshakeResponse.partyKey;
 
-    const feedKey = await this._feedInitializer(partyKey);
-
     const credentialMessages = [];
     if (haloInvitation) {
       assert(this._identity.deviceKey, 'Device key required');
@@ -164,16 +159,6 @@ export class GreetingInitiator {
           [],
           nonce)
       );
-
-      // And Feed, signed for by the FEED and the DEVICE.
-      credentialMessages.push(
-        createFeedAdmitMessage(
-          this._identity.signer,
-          partyKey,
-          feedKey,
-          [this._identity.deviceKey],
-          nonce)
-      );
     } else {
       assert(this._identity.deviceKeyChain, 'Device key required');
       assert(this._identity.identityGenesis, 'Identity genesis message required');
@@ -187,16 +172,6 @@ export class GreetingInitiator {
           [this._identity.deviceKeyChain],
           nonce)
       );
-
-      // And the Feed, signed for by the FEED and by the DEVICE keychain, as above.
-      credentialMessages.push(
-        createFeedAdmitMessage(
-          this._identity.signer,
-          partyKey,
-          feedKey,
-          [this._identity.deviceKeyChain],
-          nonce)
-      );
     }
 
     // Send the signed payload to the greeting responder.

package/src/invitations/halo-recovery-initiator.ts

@@ -7,7 +7,6 @@ import debug from 'debug';
 
 import { waitForEvent } from '@dxos/async';
 import {
-  Authenticator,
   ClaimResponse,
   Keyring,
   KeyType,
@@ -17,7 +16,8 @@ import {
   createGreetingClaimMessage,
   SecretProvider,
   SecretValidator,
-  SignedMessage
+  SignedMessage,
+  codec
 } from '@dxos/credentials';
 import { keyToBuffer, keyToString, PublicKey, randomBytes, verify } from '@dxos/crypto';
 import { raise } from '@dxos/debug';
@@ -144,7 +144,7 @@ export class HaloRecoveryInitiator {
 
   // The secretProvider should provide an `Auth` message signed directly by the Identity key.
   createSecretProvider (): SecretProvider {
-    return async (info: any) => Buffer.from(Authenticator.encodePayload(
+    return async (info: any) => Buffer.from(codec.encode(
       /* The signed portion of the Auth message includes the ID and authNonce provided
        * by "info". These values will be validated on the other end.
        */
@@ -182,7 +182,7 @@ export class HaloRecoveryInitiator {
       });
 
       const secretValidator: SecretValidator = async (invitation, secret) => {
-        const { payload: authMessage } = Authenticator.decodePayload(secret);
+        const { payload: authMessage } = codec.decode(secret);
 
         return keyring.verify(<unknown>authMessage as SignedMessage) &&
           authMessage.signed.payload.partyKey.equals(invitation.id) &&

package/src/invitations/offline-invitation-claimer.ts

@@ -7,7 +7,6 @@ import debug from 'debug';
 
 import { waitForEvent } from '@dxos/async';
 import {
-  Authenticator,
   ClaimResponse,
   Keyring,
   KeyType,
@@ -18,7 +17,8 @@ import {
   SecretInfo,
   SecretProvider,
   SecretValidator,
-  SignedMessage
+  SignedMessage,
+  codec
 } from '@dxos/credentials';
 import { keyToBuffer, keyToString, PublicKey, randomBytes } from '@dxos/crypto';
 import { raise } from '@dxos/debug';
@@ -156,7 +156,7 @@ export class OfflineInvitationClaimer {
       });
 
       const secretValidator: SecretValidator = async (invitation, secret) => {
-        const { payload: authMessage } = Authenticator.decodePayload(secret);
+        const { payload: authMessage } = codec.decode(secret);
 
         return keyring.verify(<unknown>authMessage as SignedMessage) &&
           authMessage.signed.payload.partyKey.equals(invitation.id) &&
@@ -172,7 +172,7 @@ export class OfflineInvitationClaimer {
   // The secretProvider should provide an `Auth` message signed directly by the Identity key.
   static createSecretProvider (identity: Identity): SecretProvider {
     return async (info?: SecretInfo) => {
-      return Buffer.from(Authenticator.encodePayload(
+      return Buffer.from(codec.encode(
         /* The signed portion of the Auth message includes the ID and authNonce provided
          * by the `info` object. These values will be validated on the other end.
          */

package/src/parties/authenticator.ts

@@ -0,0 +1,31 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import debug from 'debug';
+
+import { Authenticator, createEnvelopeMessage, PartyAuthenticator } from '@dxos/credentials';
+
+import { IdentityProvider } from '../halo';
+import { PartyProcessor } from '../pipeline';
+
+const log = debug('dxos:echo-db:authenticator');
+
+export function createAuthenticator (partyProcessor: PartyProcessor, identityProvider: IdentityProvider): Authenticator {
+  return new PartyAuthenticator(partyProcessor.state, async auth => {
+    if (auth.feedAdmit && auth.feedKey && !partyProcessor.isFeedAdmitted(auth.feedKey)) {
+      const deviceKeyChain = identityProvider().deviceKeyChain ?? identityProvider().deviceKey;
+      if (!deviceKeyChain) {
+        log('Not device key chain available to admit new member feed');
+        return;
+      }
+
+      await partyProcessor.writeHaloMessage(createEnvelopeMessage(
+        identityProvider().keyring,
+        partyProcessor.partyKey,
+        auth.feedAdmit,
+        [deviceKeyChain]
+      ));
+    }
+  });
+}

package/src/parties/party-core.ts

@@ -122,6 +122,7 @@ export class PartyCore {
     // Pipeline
     //
 
+    await this._feedProvider.openKnownFeeds();
     const iterator = await this._feedProvider.createIterator(
       createMessageSelector(this._partyProcessor, this._timeframeClock),
       this._initialTimeframe

package/src/parties/party-factory.ts

@@ -205,12 +205,7 @@ export class PartyFactory {
     const initiator = new GreetingInitiator(
       this._networkManager,
       identity,
-      invitationDescriptor,
-      async partyKey => {
-        const feedProvider = this._createFeedProvider(partyKey);
-        const feed = await feedProvider.createOrOpenWritableFeed();
-        return feed.key;
-      }
+      invitationDescriptor
     );
 
     await initiator.connect();

package/src/parties/party-internal.ts

@@ -5,7 +5,7 @@
 import assert from 'assert';
 
 import { synchronized, Event } from '@dxos/async';
-import { KeyHint, createAuthMessage, Authenticator } from '@dxos/credentials';
+import { KeyHint, createAuthMessage, createFeedAdmitMessage, codec } from '@dxos/credentials';
 import { PublicKey } from '@dxos/crypto';
 import { failUndefined, raise, timed } from '@dxos/debug';
 import { PartyKey, PartySnapshot, Timeframe, FeedKey } from '@dxos/echo-protocol';
@@ -19,6 +19,7 @@ import { ActivationOptions, PartyPreferences, IdentityProvider } from '../halo';
 import { InvitationManager } from '../invitations';
 import { CredentialsProvider, PartyFeedProvider, PartyProtocolFactory } from '../pipeline';
 import { SnapshotStore } from '../snapshots';
+import { createAuthenticator } from './authenticator';
 import { PartyCore, PartyOptions } from './party-core';
 import { CONTACT_DEBOUNCE_INTERVAL } from './party-manager';
 
@@ -78,7 +79,7 @@ export class PartyInternal {
       key: this.key.toHex(),
       isOpen: this.isOpen,
       isActive: this.isActive,
-      feedKeys: this._feedProvider.getFeedKeys().length,
+      feedKeys: this._feedProvider.getFeeds().length,
       timeframe: this.isOpen ? this._partyCore.timeframe : undefined,
       properties: this.isOpen ? this.getPropertiesSet().expectOne().model.toObject() : undefined
     };
@@ -178,7 +179,7 @@ export class PartyInternal {
       this._identityProvider,
       this._createCredentialsProvider(this._partyCore.key, writeFeed.key),
       this._invitationManager,
-      this._partyCore.processor.authenticator,
+      createAuthenticator(this._partyCore.processor, this._identityProvider),
       this._partyCore.processor.getActiveFeedSet()
     );
 
@@ -274,12 +275,20 @@ export class PartyInternal {
     return {
       get: () => {
         const identity = this._identityProvider();
-        return Buffer.from(Authenticator.encodePayload(createAuthMessage(
+        const signingKey = identity.deviceKeyChain ?? identity.deviceKey ?? raise(new IdentityNotInitializedError());
+        return Buffer.from(codec.encode(createAuthMessage(
           identity.signer,
           partyKey,
           identity.identityKey ?? raise(new IdentityNotInitializedError()),
-          identity.deviceKeyChain ?? identity.deviceKey ?? raise(new IdentityNotInitializedError()),
-          identity.keyring.getKey(feedKey)
+          signingKey,
+          identity.keyring.getKey(feedKey),
+          undefined,
+          createFeedAdmitMessage(
+            identity.signer,
+            partyKey,
+            feedKey,
+            [identity.keyring.getKey(feedKey) ?? failUndefined(), signingKey]
+          )
         )));
       }
     };

package/src/pipeline/party-feed-provider.ts

@@ -5,10 +5,12 @@
 import assert from 'assert';
 import debug from 'debug';
 
+import { Event } from '@dxos/async';
 import { Keyring, KeyType } from '@dxos/credentials';
 import { PublicKey } from '@dxos/crypto';
 import { FeedStoreIterator, MessageSelector, Timeframe } from '@dxos/echo-protocol';
 import { FeedDescriptor, FeedStore } from '@dxos/feed-store';
+import { ComplexMap } from '@dxos/util';
 
 import { MetadataStore } from '../metadata';
 
@@ -16,6 +18,9 @@ const STALL_TIMEOUT = 1000;
 const warn = debug('dxos:echo-db:party-feed-provider:warn');
 
 export class PartyFeedProvider {
+  private readonly _feeds = new ComplexMap<PublicKey, FeedDescriptor>(x => x.toHex())
+  readonly feedOpened = new Event<FeedDescriptor>();
+
   constructor (
     private readonly _metadataStore: MetadataStore,
     private readonly _keyring: Keyring,
@@ -23,14 +28,13 @@ export class PartyFeedProvider {
     private readonly _partyKey: PublicKey
   ) {}
 
-  // TODO(dmaretskyi): Consider refactoring this to have write feed stored separeately in metadata.
+  getFeeds (): FeedDescriptor[] {
+    return Array.from(this._feeds.values());
+  }
+
   async createOrOpenWritableFeed () {
     const partyMetadata = this._metadataStore.getParty(this._partyKey);
-    if (!partyMetadata) {
-      return this._createReadWriteFeed();
-    }
-
-    if (!partyMetadata.dataFeedKey) {
+    if (!partyMetadata?.dataFeedKey) {
       return this._createReadWriteFeed();
     }
 
@@ -39,22 +43,42 @@ export class PartyFeedProvider {
       return this._createReadWriteFeed();
     }
 
+    if (this._feeds.has(fullKey.publicKey)) {
+      return this._feeds.get(fullKey.publicKey)!;
+    }
+
     const feed = await this._feedStore.openReadWriteFeed(fullKey.publicKey, fullKey.secretKey);
-    const feedKey = this._keyring.getKey(feed.key);
-    assert(feedKey, 'Feed key not found');
+    this._feeds.set(fullKey.publicKey, feed);
+    this.feedOpened.emit(feed);
     return feed;
   }
 
-  getFeedKeys () {
-    return this._metadataStore.getParty(this._partyKey)?.feedKeys ?? [];
+  async openKnownFeeds () {
+    for (const feedKey of this._metadataStore.getParty(this._partyKey)?.feedKeys ?? []) {
+      if (!this._feeds.has(feedKey)) {
+        const fullKey = this._keyring.getFullKey(feedKey);
+        const feed = fullKey?.secretKey
+          ? await this._feedStore.openReadWriteFeed(fullKey.publicKey, fullKey.secretKey)
+          : await this._feedStore.openReadOnlyFeed(feedKey);
+        this._feeds.set(feedKey, feed);
+        this.feedOpened.emit(feed);
+      }
+    }
   }
 
   async createOrOpenReadOnlyFeed (feedKey: PublicKey): Promise<FeedDescriptor> {
+    if (this._feeds.has(feedKey)) {
+      return this._feeds.get(feedKey)!;
+    }
+
     await this._metadataStore.addPartyFeed(this._partyKey, feedKey);
     if (!this._keyring.hasKey(feedKey)) {
       await this._keyring.addPublicKey({ type: KeyType.FEED, publicKey: feedKey });
     }
-    return this._feedStore.openReadOnlyFeed(feedKey);
+    const feed = await this._feedStore.openReadOnlyFeed(feedKey);
+    this._feeds.set(feedKey, feed);
+    this.feedOpened.emit(feed);
+    return feed;
   }
 
   private async _createReadWriteFeed () {
@@ -63,16 +87,18 @@ export class PartyFeedProvider {
     assert(fullKey && fullKey.secretKey);
     await this._metadataStore.setDataFeed(this._partyKey, fullKey.publicKey);
     const feed = await this._feedStore.openReadWriteFeed(fullKey.publicKey, fullKey.secretKey);
+    this._feeds.set(fullKey.publicKey, feed);
+    this.feedOpened.emit(feed);
     return feed;
   }
 
   async createIterator (messageSelector: MessageSelector, initialTimeframe?: Timeframe) {
     const iterator = new FeedStoreIterator(() => true, messageSelector, initialTimeframe ?? new Timeframe());
-    for (const feedKey of this.getFeedKeys()) {
-      iterator.addFeedDescriptor(await this.createOrOpenReadOnlyFeed(feedKey));
+    for (const feed of this._feeds.values()) {
+      iterator.addFeedDescriptor(feed);
     }
 
-    this._feedStore.feedOpenedEvent.on((descriptor) => {
+    this.feedOpened.on((descriptor) => {
       if (this._metadataStore.getParty(this._partyKey)?.feedKeys?.find(feedKey => feedKey.equals(descriptor.key))) {
         iterator.addFeedDescriptor(descriptor);
       }

package/src/pipeline/party-processor.ts

@@ -7,11 +7,9 @@ import debug from 'debug';
 
 import { Event } from '@dxos/async';
 import {
-  Authenticator,
   KeyHint,
   KeyRecord,
   PartyState,
-  PartyAuthenticator,
   Message as HaloMessage,
   IdentityEventType,
   PartyEventType
@@ -32,7 +30,6 @@ export interface FeedSetProvider {
  */
 export class PartyProcessor {
   private readonly _state: PartyState;
-  private readonly _authenticator: Authenticator;
 
   private _outboundHaloStream: FeedWriter<HaloMessage> | undefined;
 
@@ -49,21 +46,14 @@ export class PartyProcessor {
     private readonly _partyKey: PartyKey
   ) {
     this._state = new PartyState(this._partyKey);
-    this._authenticator = new PartyAuthenticator(this._state);
-
-    /* TODO(telackey): `@dxos/credentials` was only half converted to TS. In its current state, the KeyRecord type
-     * is not exported, and the PartyStateMachine being used is not properly understood as an EventEmitter by TS.
-     * Casting to 'any' is a workaround for the compiler, but the fix is fully to convert @dxos/credentials to TS.
-     */
-    const state = this._state as any;
 
     // TODO(marik-d): Use `Event.wrap` here.
-    state.on(PartyEventType.ADMIT_FEED, (keyRecord: any) => {
+    this._state.on(PartyEventType.ADMIT_FEED, (keyRecord: any) => {
       log(`Feed key admitted ${keyRecord.publicKey.toHex()}`);
       this._feedAdded.emit(keyRecord.publicKey);
     });
-    state.on(PartyEventType.ADMIT_KEY, (keyRecord: KeyRecord) => this.keyOrInfoAdded.emit(keyRecord.publicKey));
-    state.on(IdentityEventType.UPDATE_IDENTITY, (publicKey: PublicKey) => this.keyOrInfoAdded.emit(publicKey));
+    this._state.on(PartyEventType.ADMIT_KEY, (keyRecord: KeyRecord) => this.keyOrInfoAdded.emit(keyRecord.publicKey));
+    this._state.on(IdentityEventType.UPDATE_IDENTITY, (publicKey: PublicKey) => this.keyOrInfoAdded.emit(publicKey));
   }
 
   get partyKey () {
@@ -90,8 +80,8 @@ export class PartyProcessor {
     return this._state.credentialMessages.size === 0;
   }
 
-  get authenticator () {
-    return this._authenticator;
+  get state () {
+    return this._state;
   }
 
   isFeedAdmitted (feedKey: FeedKey) {