@dxos/echo-db 2.33.5-dev.22471d71 → 2.33.5-dev.33d2877e

package/src/parties/party-factory.ts

@@ -9,7 +9,6 @@ import {
   createEnvelopeMessage,
   createFeedAdmitMessage,
   createPartyGenesisMessage,
-  KeyHint,
   KeyType,
   SecretProvider,
   wrapMessage
@@ -21,11 +20,11 @@ import { ModelFactory } from '@dxos/model-factory';
 import { NetworkManager } from '@dxos/network-manager';
 import { ObjectModel } from '@dxos/object-model';
 
-import { IdentityNotInitializedError } from '../errors';
 import {
   createDataPartyAdmissionMessages,
   GreetingInitiator, InvitationDescriptor, InvitationDescriptorType, OfflineInvitationClaimer
 } from '../invitations';
+import { IdentityNotInitializedError } from '../packlets/errors';
 import { PartyFeedProvider, PartyOptions } from '../pipeline';
 import { IdentityCredentialsProvider } from '../protocol/identity-credentials';
 import { SnapshotStore } from '../snapshots';
@@ -60,10 +59,10 @@ export class PartyFactory {
     // Connect the pipeline.
     await party.open();
 
-    const writableFeed = await party.feedProvider.createOrOpenWritableFeed();
+    const writableFeed = await party.getWriteFeed();
 
     // PartyGenesis (self-signed by Party).
-    await party.processor.writeHaloMessage(createPartyGenesisMessage(
+    await party.credentialsWriter.write(createPartyGenesisMessage(
       identity.keyring,
       partyKey,
       writableFeed.key,
@@ -71,7 +70,7 @@ export class PartyFactory {
     );
 
     // KeyAdmit (IdentityGenesis in an Envelope signed by Party).
-    await party.processor.writeHaloMessage(createEnvelopeMessage(
+    await party.credentialsWriter.write(createEnvelopeMessage(
       identity.keyring,
       partyKey.publicKey,
       wrapMessage(identity.identityGenesis),
@@ -80,7 +79,7 @@ export class PartyFactory {
 
     // FeedAdmit (signed by the Device KeyChain).
     // TODO(dmaretskyi): Is this really needed since a feed is already admitted by party genesis message.
-    await party.processor.writeHaloMessage(createFeedAdmitMessage(
+    await party.credentialsWriter.write(createFeedAdmitMessage(
       identity.keyring,
       partyKey.publicKey,
       writableFeed.key,
@@ -89,7 +88,7 @@ export class PartyFactory {
 
     // IdentityInfo in an Envelope signed by the Device KeyChain.
     if (identity.identityInfo) {
-      await party.processor.writeHaloMessage(createEnvelopeMessage(
+      await party.credentialsWriter.write(createEnvelopeMessage(
         identity.keyring,
         partyKey.publicKey,
         wrapMessage(identity.identityInfo),
@@ -106,56 +105,12 @@ export class PartyFactory {
     return party;
   }
 
-  /**
-   * Constructs a party object and creates a local write feed for it.
-   * @param partyKey
-   * @param hints
-   */
-  async addParty (partyKey: PartyKey, hints: KeyHint[] = []) {
-    const identity = this._identityProvider() ?? raise(new IdentityNotInitializedError());
-
-    /*
-     * TODO(telackey): We shouldn't have to add our key here, it should be in the hints, but our hint
-     * mechanism is broken by not waiting on the messages to be processed before returning.
-     */
-
-    const feedProvider = this._feedProviderFactory(partyKey);
-    const { feed } = await feedProvider.createOrOpenWritableFeed();
-    const feedKeyPair = identity.keyring.getKey(feed.key);
-    assert(feedKeyPair, 'Keypair for writable feed not found.');
-    const party = new DataParty(
-      partyKey,
-      this._modelFactory,
-      this._snapshotStore,
-      feedProvider,
-      identity.createCredentialsSigner(),
-      identity.preferences,
-      this._networkManager,
-      hints,
-      undefined,
-      this._options
-    );
-
-    await party.open();
-    const isHalo = identity.identityKey.publicKey.equals(partyKey);
-    const signingKey = isHalo ? identity.deviceKey : identity.deviceKeyChain;
-    assert(signingKey, 'No device key or keychain.');
-    // Write the Feed genesis message.
-    await party.processor.writeHaloMessage(createFeedAdmitMessage(
-      identity.keyring,
-      partyKey,
-      feedKeyPair.publicKey,
-      [signingKey]
-    ));
-    return party;
-  }
-
   /**
    * Constructs a party object from an existing set of feeds.
    * @param partyKey
    * @param hints
    */
-  async constructParty (partyKey: PartyKey, hints: KeyHint[] = [], initialTimeframe?: Timeframe) {
+  async constructParty (partyKey: PartyKey, feedHints: PublicKey[] = [], initialTimeframe?: Timeframe) {
     const identity = this._identityProvider() ?? raise(new IdentityNotInitializedError());
 
     // TODO(marik-d): Support read-only parties if this feed doesn't exist?
@@ -172,7 +127,7 @@ export class PartyFactory {
       identity.createCredentialsSigner(),
       identity.preferences,
       this._networkManager,
-      hints,
+      feedHints,
       initialTimeframe,
       this._options
     );
@@ -190,7 +145,6 @@ export class PartyFactory {
   }
 
   async joinParty (invitationDescriptor: InvitationDescriptor, secretProvider: SecretProvider): Promise<DataParty> {
-    const haloInvitation = !!invitationDescriptor.identityKey;
     const originalInvitation = invitationDescriptor;
 
     const identity = this._identityProvider() ?? raise(new IdentityNotInitializedError());
@@ -218,19 +172,18 @@ export class PartyFactory {
 
     await initiator.connect();
     const { partyKey, hints } = await initiator.redeemInvitation(secretProvider);
-    const party = await this.addParty(partyKey, hints);
+    const party = await this.constructParty(partyKey, hints);
+    await party.open();
     await initiator.destroy();
-    if (!haloInvitation) {
-      // Copy our signed IdentityInfo into the new Party.
-      const infoMessage = identity.identityInfo;
-      if (infoMessage) {
-        await party.processor.writeHaloMessage(createEnvelopeMessage(
-          identity.keyring,
-          partyKey,
-          wrapMessage(infoMessage),
-          [identity.deviceKeyChain]
-        ));
-      }
+
+    // Copy our signed IdentityInfo into the new Party.
+    if (identity.identityInfo) {
+      await party.credentialsWriter.write(createEnvelopeMessage(
+        identity.keyring,
+        partyKey,
+        wrapMessage(identity.identityInfo),
+        [identity.deviceKeyChain]
+      ));
     }
 
     return party;
@@ -247,10 +200,10 @@ export class PartyFactory {
     // Connect the pipeline.
     await party.open();
 
-    const writableFeed = await party.feedProvider.createOrOpenWritableFeed();
+    const writableFeed = await party.getWriteFeed();
 
     // PartyGenesis (self-signed by Party).
-    await party.processor.writeHaloMessage(createPartyGenesisMessage(
+    await party.credentialsWriter.write(createPartyGenesisMessage(
       identity.keyring,
       partyKey,
       writableFeed.key,
@@ -258,7 +211,7 @@ export class PartyFactory {
     );
 
     // KeyAdmit (IdentityGenesis in an Envelope signed by Party).
-    await party.processor.writeHaloMessage(createEnvelopeMessage(
+    await party.credentialsWriter.write(createEnvelopeMessage(
       identity.keyring,
       partyKey.publicKey,
       wrapMessage(identity.identityGenesis),
@@ -266,7 +219,7 @@ export class PartyFactory {
     ));
 
     // FeedAdmit (signed by the Device KeyChain).
-    await party.processor.writeHaloMessage(createFeedAdmitMessage(
+    await party.credentialsWriter.write(createFeedAdmitMessage(
       identity.keyring,
       partyKey.publicKey,
       writableFeed.key,
@@ -275,7 +228,7 @@ export class PartyFactory {
 
     // IdentityInfo in an Envelope signed by the Device KeyChain.
     if (identity.identityInfo) {
-      await party.processor.writeHaloMessage(createEnvelopeMessage(
+      await party.credentialsWriter.write(createEnvelopeMessage(
         identity.keyring,
         partyKey.publicKey,
         wrapMessage(identity.identityInfo),
@@ -285,7 +238,7 @@ export class PartyFactory {
 
     // const keyAdmitMessage = snapshot.halo?.messages?.[1];
     // assert(keyAdmitMessage);
-    // await party.processor.writeHaloMessage(createEnvelopeMessage(
+    // await party.writeCredentialsMessage(createEnvelopeMessage(
     //   identity.signer,
     //   partyKey.publicKey,
     //   keyAdmitMessage,
@@ -293,13 +246,14 @@ export class PartyFactory {
     // ));
 
     // for(const message of snapshot.halo?.messages?.slice(2) || []) {
-    //   await party.processor.writeHaloMessage(message);
+    //   await party.writeCredentialsMessage(message);
     // }
 
     // Write messages to create ECHO items.
     const feedWriter = createFeedWriter(writableFeed.feed);
     for (const item of snapshot.database?.items || []) {
       const message: FeedMessage = {
+        timeframe: new Timeframe(),
         echo: {
           itemId: item.itemId ?? failUndefined(),
           genesis: {
@@ -310,8 +264,7 @@ export class PartyFactory {
           itemMutation: {
             parentId: item.parentId
           },
-          snapshot: item.model,
-          timeframe: new Timeframe()
+          snapshot: item.model
         }
       };
       await feedWriter.write(message);

package/src/parties/party-manager.test.ts

@@ -32,8 +32,8 @@ import { ObjectModel } from '@dxos/object-model';
 import { createStorage, StorageType } from '@dxos/random-access-multi-storage';
 import { afterTest, testTimeout } from '@dxos/testutils';
 
-import { Item } from '../api';
 import { defaultInvitationAuthenticator, OfflineInvitationClaimer } from '../invitations';
+import { Item } from '../packlets/database';
 import { MetadataStore, PartyFeedProvider } from '../pipeline';
 import { createTestIdentityCredentials } from '../protocol/identity-credentials';
 import { SnapshotStore } from '../snapshots';
@@ -55,9 +55,11 @@ const log = debug('dxos:echo:parties:party-manager:test');
  */
 const setup = async () => {
   const keyring = new Keyring();
-  const metadataStore = new MetadataStore(createStorage('metadata', StorageType.RAM));
-  const feedStore = new FeedStore(createStorage('feed', StorageType.RAM), { valueEncoding: codec });
-  const snapshotStore = new SnapshotStore(createStorage('snapshots', StorageType.RAM));
+
+  const storage = createStorage('', StorageType.RAM);
+  const snapshotStore = new SnapshotStore(storage.directory('snapshots'));
+  const metadataStore = new MetadataStore(storage.directory('metadata'));
+  const feedStore = new FeedStore(storage.directory('feed'), { valueEncoding: codec });
   const modelFactory = new ModelFactory().registerModel(ObjectModel);
   const networkManager = new NetworkManager();
   const feedProviderFactory = (partyKey: PublicKey) => new PartyFeedProvider(metadataStore, keyring, feedStore, partyKey);
@@ -138,21 +140,18 @@ describe('Party manager', () => {
     const feedStream = createWritableFeedStream(feed);
     feedStream.write(createPartyGenesisMessage(keyring, partyKey, feedKey.publicKey, identityKey));
 
-    await partyManager.addParty(partyKey.publicKey, [{
-      type: KeyType.FEED,
-      publicKey: PublicKey.from(feed.key)
-    }]);
+    await partyManager.addParty(partyKey.publicKey, [PublicKey.from(feed.key)]);
 
     await update;
   });
 
   test('Create from cold start', async () => {
     const storage = createStorage('', StorageType.RAM);
-    const feedStore = new FeedStore(storage, { valueEncoding: codec });
+    const feedStore = new FeedStore(storage.directory('feed'), { valueEncoding: codec });
     const keyring = new Keyring();
-    const metadataStore = new MetadataStore(createStorage('metadata', StorageType.RAM));
+    const snapshotStore = new SnapshotStore(storage.directory('snapshots'));
+    const metadataStore = new MetadataStore(storage.directory('metadata'));
     const modelFactory = new ModelFactory().registerModel(ObjectModel);
-    const snapshotStore = new SnapshotStore(createStorage('snapshots', StorageType.RAM));
     const networkManager = new NetworkManager();
     const feedProviderFactory = (partyKey: PublicKey) => new PartyFeedProvider(metadataStore, keyring, feedStore, partyKey);
 
@@ -188,15 +187,23 @@ describe('Party manager', () => {
       assert(feedKey);
 
       const feedStream = createWritableFeedStream(feed);
-      feedStream.write({ halo: createPartyGenesisMessage(keyring, partyKey, feedKey.publicKey, identity.identityKey) });
       feedStream.write({
+        timeframe: new Timeframe(),
+        halo: createPartyGenesisMessage(
+          keyring,
+          partyKey,
+          feedKey.publicKey,
+          identity.identityKey
+        )
+      });
+      feedStream.write({
+        timeframe: new Timeframe(),
         echo: checkType<EchoEnvelope>({
           itemId: 'foo',
           genesis: {
             itemType: PARTY_ITEM_TYPE,
             modelType: ObjectModel.meta.type
-          },
-          timeframe: new Timeframe()
+          }
         })
       });
     }

package/src/parties/party-manager.ts

@@ -157,11 +157,9 @@ export class PartyManager {
 
   /**
    * Construct a party object and start replicating with the remote peer that created that party.
-   * @param partyKey
-   * @param hints
    */
   @synchronized
-  async addParty (partyKey: PartyKey, hints: KeyHint[] = []) {
+  async addParty (partyKey: PartyKey, feedHints: PublicKey[] = []) {
     assert(this._open, 'PartyManager is not open.');
 
     /*
@@ -174,8 +172,9 @@ export class PartyManager {
       return this._parties.get(partyKey);
     }
 
-    log(`Adding party partyKey=${partyKey.toHex()} hints=${hints.length}`);
-    const party = await this._partyFactory.addParty(partyKey, hints);
+    log(`Adding party partyKey=${partyKey.toHex()} hints=${feedHints.length}`);
+    const party = await this._partyFactory.constructParty(partyKey, feedHints);
+    await party.open();
     await this._metadataStore.addParty(party.key);
     this._setParty(party);
     return party;

package/src/pipeline/message-selector.ts

@@ -9,8 +9,8 @@ import { getPartyCredentialMessageType, PartyCredential } from '@dxos/credential
 import { PublicKey } from '@dxos/crypto';
 import { MessageSelector } from '@dxos/echo-protocol';
 
-import { TimeframeClock } from '../database';
-import { PartyProcessor } from './party-processor';
+import { TimeframeClock } from '../packlets/database';
+import { PartyStateProvider } from './party-processor';
 
 const log = debug('dxos:echo-db:message-selector');
 
@@ -23,47 +23,44 @@ const log = debug('dxos:echo-db:message-selector');
  * @param partyProcessor
  * @param timeframeClock
  */
-export function createMessageSelector (
-  partyProcessor: PartyProcessor,
-  timeframeClock: TimeframeClock
-): MessageSelector {
-  // TODO(telackey): Add KeyAdmit checks.
-  return candidates => {
-    // Check ECHO message candidates first since they are less expensive than HALO cancidates.
-    for (let i = 0; i < candidates.length; i++) {
-      const { data: { echo } } = candidates[i];
-      const feedKey = PublicKey.from(candidates[i].key);
-      if (!echo) {
-        continue;
-      }
+export const createMessageSelector = (partyProcessor: PartyStateProvider, timeframeClock: TimeframeClock): MessageSelector => candidates => {
+  // Check ECHO message candidates first since they are less expensive than HALO cancidates.
+  for (let i = 0; i < candidates.length; i++) {
+    const { data: { timeframe, echo } } = candidates[i];
+    const feedKey = PublicKey.from(candidates[i].key);
+    if (!echo) {
+      continue;
+    }
 
-      assert(echo.timeframe);
-      if (partyProcessor.isFeedAdmitted(feedKey) && !timeframeClock.hasGaps(echo.timeframe)) {
-        return i;
-      }
+    assert(timeframe);
+    if (partyProcessor.isFeedAdmitted(feedKey) && !timeframeClock.hasGaps(timeframe)) {
+      return i;
     }
+  }
 
-    // Check HALO message candidates.
-    for (let i = 0; i < candidates.length; i++) {
-      const { data: { halo } } = candidates[i];
-      const feedKey = PublicKey.from(candidates[i].key);
-      if (!halo) {
-        continue;
-      }
+  // Check HALO message candidates.
+  for (let i = 0; i < candidates.length; i++) {
+    const { data: { timeframe, halo } } = candidates[i];
+    const feedKey = PublicKey.from(candidates[i].key);
+    if (!halo) {
+      continue;
+    }
 
-      if (partyProcessor.isFeedAdmitted(feedKey)) {
-        return i;
-      }
+    assert(timeframe);
+    if (partyProcessor.isFeedAdmitted(feedKey) && !timeframeClock.hasGaps(timeframe)) {
+      return i;
+    }
 
-      if (partyProcessor.genesisRequired) {
+    if (partyProcessor.genesisRequired) {
+      try { // TODO(dmaretskyi): Get getPartyCredentialMessageType crashes for some reason.
         // TODO(telackey): Add check that this is for the right Party.
         if (getPartyCredentialMessageType(halo) === PartyCredential.Type.PARTY_GENESIS) {
           return i;
         }
-      }
+      } catch { }
     }
+  }
 
-    // Not ready for this message yet.
-    log('Skipping...');
-  };
-}
+  // Not ready for this message yet.
+  log('Skipping...');
+};

package/src/pipeline/metadata-store.test.ts

@@ -11,7 +11,8 @@ import { MetadataStore } from './metadata-store';
 
 describe('MetadataStore in-memory', () => {
   it('Creates party and adds feeds to it', async () => {
-    const store = new MetadataStore(createStorage('metadata', StorageType.RAM));
+    const storage = createStorage('', StorageType.RAM);
+    const store = new MetadataStore(storage.directory('metadata'));
     await store.load();
     expect(store.parties?.length).toBe(0);
 
@@ -33,7 +34,8 @@ describe('MetadataStore in-memory', () => {
   });
 
   it('Creates party when adding feed', async () => {
-    const store = new MetadataStore(createStorage('metadata', StorageType.RAM));
+    const storage = createStorage('', StorageType.RAM);
+    const store = new MetadataStore(storage.directory('metadata'));
     await store.load();
 
     const partyKey = PublicKey.random();
@@ -45,7 +47,8 @@ describe('MetadataStore in-memory', () => {
   });
 
   it('Doesn\'t add same feed twice', async () => {
-    const store = new MetadataStore(createStorage('metadata', StorageType.RAM));
+    const storage = createStorage('', StorageType.RAM);
+    const store = new MetadataStore(storage.directory('metadata'));
     await store.load();
 
     const partyKey = PublicKey.random();
@@ -60,7 +63,8 @@ describe('MetadataStore in-memory', () => {
 
   // TODO(yivlad): Doesn't work for now.
   it.skip('Resets storage', async () => {
-    const store = new MetadataStore(createStorage('snapshots', StorageType.RAM));
+    const storage = createStorage('snapshots', StorageType.RAM);
+    const store = new MetadataStore(storage.directory(''));
 
     const partyKey = PublicKey.random();
     const feedKey = PublicKey.random();

package/src/pipeline/metadata-store.ts

@@ -8,7 +8,7 @@ import debug from 'debug';
 import { PublicKey } from '@dxos/crypto';
 import { failUndefined } from '@dxos/debug';
 import { EchoMetadata, PartyMetadata, schema } from '@dxos/echo-protocol';
-import { Storage } from '@dxos/random-access-multi-storage';
+import { Directory } from '@dxos/random-access-multi-storage';
 
 /**
  * Version for the schema of the stored data as defined in dxos.echo.metadata.EchoMetadata.
@@ -28,7 +28,7 @@ export class MetadataStore {
   };
 
   constructor (
-    private readonly _storage: Storage
+    private readonly _directory: Directory
   ) {}
 
   get version (): number {
@@ -47,7 +47,7 @@ export class MetadataStore {
    * Loads metadata from persistent storage.
    */
   async load (): Promise<void> {
-    const file = this._storage.createOrOpen('EchoMetadata');
+    const file = this._directory.createOrOpen('EchoMetadata');
     try {
       const { size } = await file.stat();
       if (size === 0) {
@@ -75,7 +75,7 @@ export class MetadataStore {
       updated: new Date()
     };
 
-    const file = this._storage.createOrOpen('EchoMetadata');
+    const file = this._directory.createOrOpen('EchoMetadata');
 
     try {
       const encoded = Buffer.from(schema.getCodecForType('dxos.echo.metadata.EchoMetadata').encode(data));
@@ -90,7 +90,7 @@ export class MetadataStore {
    */
   async clear (): Promise<void> {
     log('Clearing all echo metadata...');
-    await this._storage.destroy();
+    await this._directory.delete();
   }
 
   /**