@dxos/echo-db 2.33.8 → 2.33.9-dev.65528f0b

package/src/packlets/database/timeframe-clock.ts

@@ -3,7 +3,9 @@
 //
 
 import { Event } from '@dxos/async';
-import { FeedKey, Timeframe } from '@dxos/echo-protocol';
+import { timed } from '@dxos/debug';
+import { FeedKey } from '@dxos/echo-protocol';
+import { Timeframe } from '@dxos/protocols';
 
 /**
  * Keeps state of the last timeframe that was processed by ECHO.
@@ -29,6 +31,7 @@ export class TimeframeClock {
     return !gaps.isEmpty();
   }
 
+  @timed(5_000)
   async waitUntilReached (target: Timeframe) {
     await this.update.waitForCondition(() => Timeframe.dependencies(target, this._timeframe).isEmpty());
   }

package/src/parties/data-party.test.ts

@@ -22,7 +22,7 @@ import { SnapshotStore } from '../snapshots';
 import { DataParty } from './data-party';
 
 describe('DataParty', () => {
-  const createParty = async (identity: IdentityCredentials, partyKey: PublicKey, feedHints: PublicKey[]) => {
+  const createParty = async (identity: IdentityCredentials, partyKey: PublicKey, genesisFeedKey?: PublicKey) => {
 
     const storage = createStorage('', StorageType.RAM);
     const snapshotStore = new SnapshotStore(storage.directory('snapshots'));
@@ -43,7 +43,7 @@ describe('DataParty', () => {
       identity.preferences,
       networkManager
     );
-    party._setFeedHints([...feedHints, writableFeed.key]);
+    party._setGenesisFeedKey(genesisFeedKey ?? writableFeed.key);
     return party;
   };
 
@@ -51,7 +51,7 @@ describe('DataParty', () => {
     const keyring = new Keyring();
     const identity = await createTestIdentityCredentials(keyring);
     const partyKey = await keyring.createKeyRecord({ type: KeyType.PARTY });
-    const party = await createParty(identity, partyKey.publicKey, []);
+    const party = await createParty(identity, partyKey.publicKey);
 
     await party.open();
     await party.close();
@@ -61,7 +61,7 @@ describe('DataParty', () => {
     const keyring = new Keyring();
     const identity = await createTestIdentityCredentials(keyring);
     const partyKey = await keyring.createKeyRecord({ type: KeyType.PARTY });
-    const party = await createParty(identity, partyKey.publicKey, []);
+    const party = await createParty(identity, partyKey.publicKey);
     await party.open();
 
     const feed = await party.getWriteFeed();
@@ -81,7 +81,7 @@ describe('DataParty', () => {
     const keyring = new Keyring();
     const identity = await createTestIdentityCredentials(keyring);
     const partyKey = await keyring.createKeyRecord({ type: KeyType.PARTY });
-    const party = await createParty(identity, partyKey.publicKey, []);
+    const party = await createParty(identity, partyKey.publicKey);
     await party.open();
 
     const feed = await party.getWriteFeed();
@@ -109,7 +109,7 @@ describe('DataParty', () => {
     const identity = await createTestIdentityCredentials(keyring);
     const partyKey = await keyring.createKeyRecord({ type: KeyType.PARTY });
 
-    const party = await createParty(identity, partyKey.publicKey, []);
+    const party = await createParty(identity, partyKey.publicKey);
     await party.open();
     const feed = await party.getWriteFeed();
     await party.credentialsWriter.write(createPartyGenesisMessage(
@@ -134,7 +134,7 @@ describe('DataParty', () => {
     const identityA = await createTestIdentityCredentials(keyring);
     const partyKey = await keyring.createKeyRecord({ type: KeyType.PARTY });
 
-    const party = await createParty(identityA, partyKey.publicKey, []);
+    const party = await createParty(identityA, partyKey.publicKey);
     await party.open();
     const feed = await party.getWriteFeed();
     await party.credentialsWriter.write(createPartyGenesisMessage(
@@ -161,7 +161,7 @@ describe('DataParty', () => {
     const identityA = await createTestIdentityCredentials(keyring);
     const partyKey = await keyring.createKeyRecord({ type: KeyType.PARTY });
 
-    const partyA = await createParty(identityA, partyKey.publicKey, []);
+    const partyA = await createParty(identityA, partyKey.publicKey);
     await partyA.open();
     const feedA = await partyA.getWriteFeed();
     await partyA.credentialsWriter.write(createPartyGenesisMessage(
@@ -178,7 +178,7 @@ describe('DataParty', () => {
     ));
 
     const identityB = await deriveTestDeviceCredentials(identityA);
-    const partyB = await createParty(identityB, partyKey.publicKey, [feedA.key]);
+    const partyB = await createParty(identityB, partyKey.publicKey, feedA.key);
     await partyB.open();
 
     await partyA.database.createItem({ type: 'test:item-a' });
@@ -195,7 +195,7 @@ describe('DataParty', () => {
     const identityA = await createTestIdentityCredentials(new Keyring());
     const partyKeyA = await identityA.keyring.createKeyRecord({ type: KeyType.PARTY });
 
-    const partyA = await createParty(identityA, partyKeyA.publicKey, []);
+    const partyA = await createParty(identityA, partyKeyA.publicKey);
     await partyA.open();
     const feedA = await partyA.getWriteFeed();
     await partyA.credentialsWriter.write(createPartyGenesisMessage(
@@ -226,9 +226,9 @@ describe('DataParty', () => {
     );
 
     await initiator.connect();
-    const { partyKey: partyKeyB, hints: hintsB } = await initiator.redeemInvitation(defaultSecretProvider);
+    const { partyKey: partyKeyB, genesisFeedKey } = await initiator.redeemInvitation(defaultSecretProvider);
     expect(partyKeyB.equals(partyKeyA.publicKey));
-    const partyB = await createParty(identityB, partyKeyB, hintsB);
+    const partyB = await createParty(identityB, partyKeyB, genesisFeedKey);
     await partyB.open();
     await initiator.destroy();
 

package/src/parties/data-party.ts

@@ -2,16 +2,16 @@
 // Copyright 2020 DXOS.org
 //
 
-import assert from 'assert';
+import assert from 'node:assert';
 
 import { synchronized, Event } from '@dxos/async';
 import { timed } from '@dxos/debug';
-import { PartyKey, PartySnapshot, Timeframe } from '@dxos/echo-protocol';
+import { PartyKey, PartySnapshot } from '@dxos/echo-protocol';
 import { FeedDescriptor } from '@dxos/feed-store';
 import { ModelFactory } from '@dxos/model-factory';
 import { NetworkManager } from '@dxos/network-manager';
 import { ObjectModel } from '@dxos/object-model';
-import { PublicKey } from '@dxos/protocols';
+import { PublicKey, Timeframe } from '@dxos/protocols';
 
 import { ResultSet } from '../api';
 import { ActivationOptions, PartyPreferences, Preferences } from '../halo';
@@ -46,7 +46,8 @@ export class DataParty {
   private readonly _preferences?: PartyPreferences;
   private _invitationManager?: InvitationFactory;
   private _protocol?: PartyProtocolFactory;
-  private _feedHints: PublicKey[] = []
+
+  private _genesisFeedKey?: PublicKey | undefined;
 
   constructor (
     partyKey: PartyKey,
@@ -141,11 +142,16 @@ export class DataParty {
     await this._preferences?.setLastKnownTitle(title);
   }
 
+  get genesisFeedKey () {
+    assert(this._genesisFeedKey);
+    return this._genesisFeedKey;
+  }
+
   /**
    * @internal
    */
-  _setFeedHints (feedHints: PublicKey[]) {
-    this._feedHints = feedHints;
+  _setGenesisFeedKey (genesisFeedKey: PublicKey) {
+    this._genesisFeedKey = genesisFeedKey;
   }
 
   /**
@@ -161,8 +167,9 @@ export class DataParty {
     // TODO(dmaretskyi): May be undefined in some tests.
     const party = this._metadataStore.getParty(this._partyCore.key);
 
+    assert(this._genesisFeedKey);
     await this._partyCore.open({
-      feedHints: this._feedHints,
+      genesisFeedKey: this._genesisFeedKey,
       initialTimeframe: this._initialTimeframe,
       targetTimeframe: party?.latestTimeframe
     });
@@ -175,6 +182,7 @@ export class DataParty {
 
     this._invitationManager = new InvitationFactory(
       this._partyCore.processor,
+      this._genesisFeedKey,
       this._credentialsSigner,
       this._partyCore.credentialsWriter,
       this._networkManager

package/src/parties/party-factory.ts

@@ -2,8 +2,8 @@
 // Copyright 2020 DXOS.org
 //
 
-import assert from 'assert';
 import debug from 'debug';
+import assert from 'node:assert';
 
 import {
   createEnvelopeMessage,
@@ -14,11 +14,11 @@ import {
   wrapMessage
 } from '@dxos/credentials';
 import { failUndefined, raise, timed } from '@dxos/debug';
-import { createFeedWriter, FeedMessage, PartyKey, PartySnapshot, Timeframe } from '@dxos/echo-protocol';
+import { createFeedWriter, FeedMessage, PartyKey, PartySnapshot } from '@dxos/echo-protocol';
 import { ModelFactory } from '@dxos/model-factory';
 import { NetworkManager } from '@dxos/network-manager';
 import { ObjectModel } from '@dxos/object-model';
-import { PublicKey } from '@dxos/protocols';
+import { PublicKey, Timeframe } from '@dxos/protocols';
 import { humanize } from '@dxos/util';
 
 import {
@@ -59,8 +59,10 @@ export class PartyFactory {
     const party = await this.constructParty(partyKey.publicKey);
 
     const writableFeed = await party.getWriteFeed();
-    // Hint at the newly created writable feed so that we can start replicating from it.
-    party._setFeedHints([writableFeed.key]);
+    party._setGenesisFeedKey(writableFeed.key);
+
+    await this._metadataStore.addParty(partyKey.publicKey);
+    await this._metadataStore.setGenesisFeed(partyKey.publicKey, writableFeed.key);
 
     // Connect the pipeline.
     await party.open();
@@ -111,8 +113,6 @@ export class PartyFactory {
 
   /**
    * Constructs a party object from an existing set of feeds.
-   * @param partyKey
-   * @param hints
    */
   async constructParty (partyKey: PartyKey, initialTimeframe?: Timeframe) {
     const identity = this._identityProvider() ?? raise(new IdentityNotInitializedError());
@@ -172,9 +172,14 @@ export class PartyFactory {
     );
 
     await initiator.connect();
-    const { partyKey, hints } = await initiator.redeemInvitation(secretProvider);
+    const { partyKey, genesisFeedKey } = await initiator.redeemInvitation(secretProvider);
     const party = await this.constructParty(partyKey);
-    party._setFeedHints(hints);
+
+    await this._metadataStore.addParty(partyKey);
+    await this._metadataStore.setGenesisFeed(partyKey, genesisFeedKey);
+
+    party._setGenesisFeedKey(genesisFeedKey);
+
     await party.open();
     await initiator.destroy();
 
@@ -201,7 +206,10 @@ export class PartyFactory {
 
     const writableFeed = await party.getWriteFeed();
     // Hint at the newly created writable feed so that we can start replicating from it.
-    party._setFeedHints([writableFeed.key]);
+    party._setGenesisFeedKey(writableFeed.key);
+
+    await this._metadataStore.addParty(partyKey.publicKey);
+    await this._metadataStore.setGenesisFeed(partyKey.publicKey, writableFeed.key);
 
     // Connect the pipeline.
     await party.open();

package/src/parties/party-manager.test.ts

@@ -4,10 +4,10 @@
 
 /* eslint-disable jest/no-conditional-expect */
 
-import assert from 'assert';
 import debug from 'debug';
 import expect from 'expect';
 import { it as test } from 'mocha';
+import assert from 'node:assert';
 
 import { latch } from '@dxos/async';
 import {
@@ -25,12 +25,12 @@ import {
   verify
 } from '@dxos/crypto';
 import { checkType } from '@dxos/debug';
-import { codec, EchoEnvelope, Timeframe } from '@dxos/echo-protocol';
+import { codec, EchoEnvelope } from '@dxos/echo-protocol';
 import { createWritableFeedStream, FeedStore } from '@dxos/feed-store';
 import { ModelFactory } from '@dxos/model-factory';
 import { NetworkManager } from '@dxos/network-manager';
 import { ObjectModel } from '@dxos/object-model';
-import { PublicKey } from '@dxos/protocols';
+import { PublicKey, Timeframe } from '@dxos/protocols';
 import { createStorage, StorageType } from '@dxos/random-access-multi-storage';
 import { afterTest, testTimeout } from '@dxos/testutils';
 import { humanize } from '@dxos/util';
@@ -134,17 +134,19 @@ describe('Party manager', () => {
 
     // TODO(burdon): Create multiple feeds.
     const { publicKey, secretKey } = createKeyPair();
-    const { feed } = await feedStore.openReadWriteFeed(PublicKey.from(publicKey), secretKey);
+    const feed = await feedStore.openReadWriteFeed(PublicKey.from(publicKey), secretKey);
     const feedKey = await keyring.addKeyRecord({
       publicKey: PublicKey.from(feed.key),
       secretKey: feed.secretKey,
       type: KeyType.FEED
     });
 
-    const feedStream = createWritableFeedStream(feed);
-    feedStream.write(createPartyGenesisMessage(keyring, partyKey, feedKey.publicKey, identityKey));
+    await feed.append({
+      timeframe: new Timeframe(),
+      halo: createPartyGenesisMessage(keyring, partyKey, feedKey.publicKey, identityKey)
+    });
 
-    await partyManager.addParty(partyKey.publicKey, [PublicKey.from(feed.key)]);
+    await partyManager.addParty(partyKey.publicKey, feed.key);
 
     await update;
   });
@@ -185,6 +187,7 @@ describe('Party manager', () => {
       assert(keyRecord, 'Key is not found in keyring');
       assert(keyRecord.secretKey, 'Missing secret key');
       await metadataStore.addPartyFeed(partyKey.publicKey, keyRecord.publicKey);
+      await metadataStore.setGenesisFeed(partyKey.publicKey, keyRecord.publicKey);
 
       // TODO(burdon): Create multiple feeds.
       const { feed } = await feedStore.openReadWriteFeed(keyRecord.publicKey, keyRecord.secretKey);

package/src/parties/party-manager.ts

@@ -2,12 +2,12 @@
 // Copyright 2020 DXOS.org
 //
 
-import assert from 'assert';
 import debug from 'debug';
 import unionWith from 'lodash.unionwith';
+import assert from 'node:assert';
 
 import { Event, synchronized } from '@dxos/async';
-import { KeyHint, KeyType, SecretProvider } from '@dxos/credentials';
+import { SecretProvider } from '@dxos/credentials';
 import { failUndefined, timed } from '@dxos/debug';
 import { PartyKey, PartySnapshot } from '@dxos/echo-protocol';
 import { PublicKey } from '@dxos/protocols';
@@ -96,11 +96,15 @@ export class PartyManager {
         const snapshot = await this._snapshotStore.load(partyKey);
 
         const metadata = this._metadataStore.getParty(partyKey) ?? failUndefined();
+        if (!metadata.genesisFeedKey) {
+          log(`Skipping loading party with missing genesis feed key: ${partyKey}`);
+          continue;
+        }
 
         const party = snapshot
           ? await this._partyFactory.constructPartyFromSnapshot(snapshot)
           : await this._partyFactory.constructParty(partyKey);
-        party._setFeedHints(metadata.feedKeys ?? []);
+        party._setGenesisFeedKey(metadata.genesisFeedKey);
 
         const isActive = identity?.preferences?.isPartyActive(partyKey) ?? true;
         if (isActive) {
@@ -163,7 +167,7 @@ export class PartyManager {
    * Construct a party object and start replicating with the remote peer that created that party.
    */
   @synchronized
-  async addParty (partyKey: PartyKey, feedHints: PublicKey[] = []) {
+  async addParty (partyKey: PartyKey, genesisFeedKey: PublicKey) {
     assert(this._open, 'PartyManager is not open.');
 
     /*
@@ -176,11 +180,14 @@ export class PartyManager {
       return this._parties.get(partyKey);
     }
 
-    log(`Adding party partyKey=${partyKey.toHex()} hints=${feedHints.length}`);
+    log(`Adding party partyKey=${partyKey.toHex()}`);
     const party = await this._partyFactory.constructParty(partyKey);
-    party._setFeedHints(feedHints);
-    await party.open();
+    party._setGenesisFeedKey(genesisFeedKey);
+
     await this._metadataStore.addParty(party.key);
+    await this._metadataStore.setGenesisFeed(party.key, genesisFeedKey);
+
+    await party.open();
     this._setParty(party);
     return party;
   }
@@ -327,14 +334,9 @@ export class PartyManager {
       return;
     }
 
-    const keyHints: KeyHint[] = [
-      ...party.processor.memberKeys.map(publicKey => ({ publicKey: publicKey, type: KeyType.UNKNOWN })),
-      ...party.processor.feedKeys.map(publicKey => ({ publicKey: publicKey, type: KeyType.FEED }))
-    ];
-
     await identity.preferences.recordPartyJoining({
       partyKey: party.key,
-      keyHints
+      genesisFeed: party.genesisFeedKey
     });
   }
 }

package/src/parties/party-preferences.ts

@@ -2,7 +2,7 @@
 // Copyright 2020 DXOS.org
 //
 
-import assert from 'assert';
+import assert from 'node:assert';
 
 import { PARTY_TITLE_PROPERTY, DataParty } from '.';
 import { Preferences } from '../halo/preferences';

package/src/pipeline/feed-muxer.test.ts

@@ -9,10 +9,10 @@ import { it as test } from 'mocha';
 import { waitForCondition, latch } from '@dxos/async';
 import { createPartyGenesisMessage, Keyring, KeyType } from '@dxos/credentials';
 import { createId, createKeyPair } from '@dxos/crypto';
-import { codec, createFeedWriter, FeedSelector, FeedStoreIterator, IEchoStream, Timeframe } from '@dxos/echo-protocol';
+import { codec, createFeedWriter, FeedSelector, FeedStoreIterator, IEchoStream } from '@dxos/echo-protocol';
 import { FeedStore, createWritableFeedStream } from '@dxos/feed-store';
 import { createSetPropertyMutation } from '@dxos/model-factory';
-import { PublicKey } from '@dxos/protocols';
+import { PublicKey, Timeframe } from '@dxos/protocols';
 import { createStorage, StorageType } from '@dxos/random-access-multi-storage';
 import { jsonReplacer } from '@dxos/util';
 

package/src/pipeline/feed-muxer.ts

@@ -2,16 +2,16 @@
 // Copyright 2020 DXOS.org
 //
 
-import assert from 'assert';
 import debug from 'debug';
+import assert from 'node:assert';
 
 import { Event } from '@dxos/async';
 import { Message as HaloMessage } from '@dxos/credentials';
 import { checkType } from '@dxos/debug';
 import {
-  createFeedMeta, EchoEnvelope, FeedMessage, FeedStoreIterator, FeedWriter, IEchoStream, mapFeedWriter, Timeframe
+  createFeedMeta, EchoEnvelope, FeedMessage, FeedStoreIterator, FeedWriter, IEchoStream, mapFeedWriter
 } from '@dxos/echo-protocol';
-import { PublicKey } from '@dxos/protocols';
+import { PublicKey, Timeframe } from '@dxos/protocols';
 import { jsonReplacer } from '@dxos/util';
 
 import { EchoProcessor, TimeframeClock } from '../packlets/database';

package/src/pipeline/message-selector.ts

@@ -2,15 +2,12 @@
 // Copyright 2020 DXOS.org
 //
 
-import assert from 'assert';
 import debug from 'debug';
+import assert from 'node:assert';
 
-import { getPartyCredentialMessageType, PartyCredential } from '@dxos/credentials';
 import { MessageSelector } from '@dxos/echo-protocol';
-import { PublicKey } from '@dxos/protocols';
 
 import { TimeframeClock } from '../packlets/database';
-import { PartyStateProvider } from './party-processor';
 
 const log = debug('dxos:echo-db:message-selector');
 
@@ -23,44 +20,16 @@ const log = debug('dxos:echo-db:message-selector');
  * @param partyProcessor
  * @param timeframeClock
  */
-export const createMessageSelector = (partyProcessor: PartyStateProvider, timeframeClock: TimeframeClock): MessageSelector => candidates => {
-  // Check ECHO message candidates first since they are less expensive than HALO cancidates.
+export const createMessageSelector = (timeframeClock: TimeframeClock): MessageSelector => candidates => {
+  // Pick the first candidate with a valid timeframe that has no gaps.
   for (let i = 0; i < candidates.length; i++) {
-    const { data: { timeframe, echo } } = candidates[i];
-    const feedKey = PublicKey.from(candidates[i].key);
-    if (!echo) {
-      continue;
-    }
-
-    assert(timeframe);
-    if (partyProcessor.isFeedAdmitted(feedKey) && !timeframeClock.hasGaps(timeframe)) {
-      return i;
-    }
-  }
-
-  // Check HALO message candidates.
-  for (let i = 0; i < candidates.length; i++) {
-    const { data: { timeframe, halo } } = candidates[i];
-    const feedKey = PublicKey.from(candidates[i].key);
-    if (!halo) {
-      continue;
-    }
+    const { data: { timeframe } } = candidates[i];
 
     assert(timeframe);
-    if (partyProcessor.isFeedAdmitted(feedKey) && !timeframeClock.hasGaps(timeframe)) {
+    if (!timeframeClock.hasGaps(timeframe)) {
       return i;
     }
-
-    if (partyProcessor.genesisRequired) {
-      try { // TODO(dmaretskyi): Get getPartyCredentialMessageType crashes for some reason.
-        // TODO(telackey): Add check that this is for the right Party.
-        if (getPartyCredentialMessageType(halo) === PartyCredential.Type.PARTY_GENESIS) {
-          return i;
-        }
-      } catch { }
-    }
   }
-
   // Not ready for this message yet.
   log('Skipping...');
 };

package/src/pipeline/metadata-store.ts

@@ -2,13 +2,13 @@
 // Copyright 2021 DXOS.org
 //
 
-import assert from 'assert';
 import debug from 'debug';
+import assert from 'node:assert';
 
 import { synchronized } from '@dxos/async';
 import { failUndefined } from '@dxos/debug';
-import { EchoMetadata, PartyMetadata, schema, Timeframe } from '@dxos/echo-protocol';
-import { PublicKey } from '@dxos/protocols';
+import { EchoMetadata, PartyMetadata, schema } from '@dxos/echo-protocol';
+import { PublicKey, Timeframe } from '@dxos/protocols';
 import { Directory } from '@dxos/random-access-multi-storage';
 
 /**
@@ -20,6 +20,11 @@ export const STORAGE_VERSION = 1;
 
 const log = debug('dxos:snapshot-store');
 
+export interface AddPartyOptions {
+  key: PublicKey
+  genesisFeed: PublicKey
+}
+
 export class MetadataStore {
   private _metadata: EchoMetadata = {
     version: STORAGE_VERSION,
@@ -147,6 +152,14 @@ export class MetadataStore {
     await this._save();
   }
 
+  async setGenesisFeed (partyKey: PublicKey, feedKey: PublicKey): Promise<void> {
+    assert(PublicKey.isPublicKey(feedKey));
+    await this.addPartyFeed(partyKey, feedKey);
+    const party = this.getParty(partyKey) ?? failUndefined();
+    party.genesisFeedKey = feedKey;
+    await this._save();
+  }
+
   /**
    * Sets the data feed key in the party specified by public key and saves updated data in persistent storage.
    * Update party's feed list.

package/src/pipeline/party-feed-provider.ts

@@ -2,14 +2,14 @@
 // Copyright 2021 DXOS.org
 //
 
-import assert from 'assert';
 import debug from 'debug';
+import assert from 'node:assert';
 
 import { Event, synchronized } from '@dxos/async';
 import { Keyring, KeyType } from '@dxos/credentials';
-import { FeedSelector, FeedStoreIterator, MessageSelector, Timeframe } from '@dxos/echo-protocol';
+import { FeedSelector, FeedStoreIterator, MessageSelector } from '@dxos/echo-protocol';
 import { FeedDescriptor, FeedStore } from '@dxos/feed-store';
-import { PublicKey } from '@dxos/protocols';
+import { PublicKey, Timeframe } from '@dxos/protocols';
 import { ComplexMap } from '@dxos/util';
 
 import { MetadataStore } from './metadata-store';