@dxos/echo-db 2.33.8 → 2.33.9-dev.e605934d

package/src/parties/party-manager.test.ts

@@ -25,12 +25,12 @@ import {
   verify
 } from '@dxos/crypto';
 import { checkType } from '@dxos/debug';
-import { codec, EchoEnvelope, Timeframe } from '@dxos/echo-protocol';
+import { codec, EchoEnvelope } from '@dxos/echo-protocol';
 import { createWritableFeedStream, FeedStore } from '@dxos/feed-store';
 import { ModelFactory } from '@dxos/model-factory';
 import { NetworkManager } from '@dxos/network-manager';
 import { ObjectModel } from '@dxos/object-model';
-import { PublicKey } from '@dxos/protocols';
+import { PublicKey, Timeframe } from '@dxos/protocols';
 import { createStorage, StorageType } from '@dxos/random-access-multi-storage';
 import { afterTest, testTimeout } from '@dxos/testutils';
 import { humanize } from '@dxos/util';
@@ -134,17 +134,19 @@ describe('Party manager', () => {
 
     // TODO(burdon): Create multiple feeds.
     const { publicKey, secretKey } = createKeyPair();
-    const { feed } = await feedStore.openReadWriteFeed(PublicKey.from(publicKey), secretKey);
+    const feed = await feedStore.openReadWriteFeed(PublicKey.from(publicKey), secretKey);
     const feedKey = await keyring.addKeyRecord({
       publicKey: PublicKey.from(feed.key),
       secretKey: feed.secretKey,
       type: KeyType.FEED
     });
 
-    const feedStream = createWritableFeedStream(feed);
-    feedStream.write(createPartyGenesisMessage(keyring, partyKey, feedKey.publicKey, identityKey));
+    await feed.append({
+      timeframe: new Timeframe(),
+      halo: createPartyGenesisMessage(keyring, partyKey, feedKey.publicKey, identityKey)
+    });
 
-    await partyManager.addParty(partyKey.publicKey, [PublicKey.from(feed.key)]);
+    await partyManager.addParty(partyKey.publicKey, feed.key);
 
     await update;
   });
@@ -185,6 +187,7 @@ describe('Party manager', () => {
       assert(keyRecord, 'Key is not found in keyring');
       assert(keyRecord.secretKey, 'Missing secret key');
       await metadataStore.addPartyFeed(partyKey.publicKey, keyRecord.publicKey);
+      await metadataStore.setGenesisFeed(partyKey.publicKey, keyRecord.publicKey);
 
       // TODO(burdon): Create multiple feeds.
       const { feed } = await feedStore.openReadWriteFeed(keyRecord.publicKey, keyRecord.secretKey);

package/src/parties/party-manager.ts

@@ -7,7 +7,7 @@ import debug from 'debug';
 import unionWith from 'lodash.unionwith';
 
 import { Event, synchronized } from '@dxos/async';
-import { KeyHint, KeyType, SecretProvider } from '@dxos/credentials';
+import { SecretProvider } from '@dxos/credentials';
 import { failUndefined, timed } from '@dxos/debug';
 import { PartyKey, PartySnapshot } from '@dxos/echo-protocol';
 import { PublicKey } from '@dxos/protocols';
@@ -96,11 +96,15 @@ export class PartyManager {
         const snapshot = await this._snapshotStore.load(partyKey);
 
         const metadata = this._metadataStore.getParty(partyKey) ?? failUndefined();
+        if (!metadata.genesisFeedKey) {
+          log(`Skipping loading party with missing genesis feed key: ${partyKey}`);
+          continue;
+        }
 
         const party = snapshot
           ? await this._partyFactory.constructPartyFromSnapshot(snapshot)
           : await this._partyFactory.constructParty(partyKey);
-        party._setFeedHints(metadata.feedKeys ?? []);
+        party._setGenesisFeedKey(metadata.genesisFeedKey);
 
         const isActive = identity?.preferences?.isPartyActive(partyKey) ?? true;
         if (isActive) {
@@ -163,7 +167,7 @@ export class PartyManager {
    * Construct a party object and start replicating with the remote peer that created that party.
    */
   @synchronized
-  async addParty (partyKey: PartyKey, feedHints: PublicKey[] = []) {
+  async addParty (partyKey: PartyKey, genesisFeedKey: PublicKey) {
     assert(this._open, 'PartyManager is not open.');
 
     /*
@@ -176,11 +180,14 @@ export class PartyManager {
       return this._parties.get(partyKey);
     }
 
-    log(`Adding party partyKey=${partyKey.toHex()} hints=${feedHints.length}`);
+    log(`Adding party partyKey=${partyKey.toHex()}`);
     const party = await this._partyFactory.constructParty(partyKey);
-    party._setFeedHints(feedHints);
-    await party.open();
+    party._setGenesisFeedKey(genesisFeedKey);
+
     await this._metadataStore.addParty(party.key);
+    await this._metadataStore.setGenesisFeed(party.key, genesisFeedKey);
+
+    await party.open();
     this._setParty(party);
     return party;
   }
@@ -327,14 +334,9 @@ export class PartyManager {
       return;
     }
 
-    const keyHints: KeyHint[] = [
-      ...party.processor.memberKeys.map(publicKey => ({ publicKey: publicKey, type: KeyType.UNKNOWN })),
-      ...party.processor.feedKeys.map(publicKey => ({ publicKey: publicKey, type: KeyType.FEED }))
-    ];
-
     await identity.preferences.recordPartyJoining({
       partyKey: party.key,
-      keyHints
+      genesisFeed: party.genesisFeedKey
     });
   }
 }

package/src/pipeline/feed-muxer.test.ts

@@ -9,10 +9,10 @@ import { it as test } from 'mocha';
 import { waitForCondition, latch } from '@dxos/async';
 import { createPartyGenesisMessage, Keyring, KeyType } from '@dxos/credentials';
 import { createId, createKeyPair } from '@dxos/crypto';
-import { codec, createFeedWriter, FeedSelector, FeedStoreIterator, IEchoStream, Timeframe } from '@dxos/echo-protocol';
+import { codec, createFeedWriter, FeedSelector, FeedStoreIterator, IEchoStream } from '@dxos/echo-protocol';
 import { FeedStore, createWritableFeedStream } from '@dxos/feed-store';
 import { createSetPropertyMutation } from '@dxos/model-factory';
-import { PublicKey } from '@dxos/protocols';
+import { PublicKey, Timeframe } from '@dxos/protocols';
 import { createStorage, StorageType } from '@dxos/random-access-multi-storage';
 import { jsonReplacer } from '@dxos/util';
 

package/src/pipeline/feed-muxer.ts

@@ -9,9 +9,9 @@ import { Event } from '@dxos/async';
 import { Message as HaloMessage } from '@dxos/credentials';
 import { checkType } from '@dxos/debug';
 import {
-  createFeedMeta, EchoEnvelope, FeedMessage, FeedStoreIterator, FeedWriter, IEchoStream, mapFeedWriter, Timeframe
+  createFeedMeta, EchoEnvelope, FeedMessage, FeedStoreIterator, FeedWriter, IEchoStream, mapFeedWriter
 } from '@dxos/echo-protocol';
-import { PublicKey } from '@dxos/protocols';
+import { PublicKey, Timeframe } from '@dxos/protocols';
 import { jsonReplacer } from '@dxos/util';
 
 import { EchoProcessor, TimeframeClock } from '../packlets/database';

package/src/pipeline/message-selector.ts

@@ -5,12 +5,9 @@
 import assert from 'assert';
 import debug from 'debug';
 
-import { getPartyCredentialMessageType, PartyCredential } from '@dxos/credentials';
 import { MessageSelector } from '@dxos/echo-protocol';
-import { PublicKey } from '@dxos/protocols';
 
 import { TimeframeClock } from '../packlets/database';
-import { PartyStateProvider } from './party-processor';
 
 const log = debug('dxos:echo-db:message-selector');
 
@@ -23,44 +20,16 @@ const log = debug('dxos:echo-db:message-selector');
  * @param partyProcessor
  * @param timeframeClock
  */
-export const createMessageSelector = (partyProcessor: PartyStateProvider, timeframeClock: TimeframeClock): MessageSelector => candidates => {
-  // Check ECHO message candidates first since they are less expensive than HALO cancidates.
+export const createMessageSelector = (timeframeClock: TimeframeClock): MessageSelector => candidates => {
+  // Pick the first candidate with a valid timeframe that has no gaps.
   for (let i = 0; i < candidates.length; i++) {
-    const { data: { timeframe, echo } } = candidates[i];
-    const feedKey = PublicKey.from(candidates[i].key);
-    if (!echo) {
-      continue;
-    }
-
-    assert(timeframe);
-    if (partyProcessor.isFeedAdmitted(feedKey) && !timeframeClock.hasGaps(timeframe)) {
-      return i;
-    }
-  }
-
-  // Check HALO message candidates.
-  for (let i = 0; i < candidates.length; i++) {
-    const { data: { timeframe, halo } } = candidates[i];
-    const feedKey = PublicKey.from(candidates[i].key);
-    if (!halo) {
-      continue;
-    }
+    const { data: { timeframe } } = candidates[i];
 
     assert(timeframe);
-    if (partyProcessor.isFeedAdmitted(feedKey) && !timeframeClock.hasGaps(timeframe)) {
+    if (!timeframeClock.hasGaps(timeframe)) {
       return i;
     }
-
-    if (partyProcessor.genesisRequired) {
-      try { // TODO(dmaretskyi): Get getPartyCredentialMessageType crashes for some reason.
-        // TODO(telackey): Add check that this is for the right Party.
-        if (getPartyCredentialMessageType(halo) === PartyCredential.Type.PARTY_GENESIS) {
-          return i;
-        }
-      } catch { }
-    }
   }
-
   // Not ready for this message yet.
   log('Skipping...');
 };

package/src/pipeline/metadata-store.ts

@@ -7,8 +7,8 @@ import debug from 'debug';
 
 import { synchronized } from '@dxos/async';
 import { failUndefined } from '@dxos/debug';
-import { EchoMetadata, PartyMetadata, schema, Timeframe } from '@dxos/echo-protocol';
-import { PublicKey } from '@dxos/protocols';
+import { EchoMetadata, PartyMetadata, schema } from '@dxos/echo-protocol';
+import { PublicKey, Timeframe } from '@dxos/protocols';
 import { Directory } from '@dxos/random-access-multi-storage';
 
 /**
@@ -20,6 +20,11 @@ export const STORAGE_VERSION = 1;
 
 const log = debug('dxos:snapshot-store');
 
+export interface AddPartyOptions {
+  key: PublicKey
+  genesisFeed: PublicKey
+}
+
 export class MetadataStore {
   private _metadata: EchoMetadata = {
     version: STORAGE_VERSION,
@@ -147,6 +152,14 @@ export class MetadataStore {
     await this._save();
   }
 
+  async setGenesisFeed (partyKey: PublicKey, feedKey: PublicKey): Promise<void> {
+    assert(PublicKey.isPublicKey(feedKey));
+    await this.addPartyFeed(partyKey, feedKey);
+    const party = this.getParty(partyKey) ?? failUndefined();
+    party.genesisFeedKey = feedKey;
+    await this._save();
+  }
+
   /**
    * Sets the data feed key in the party specified by public key and saves updated data in persistent storage.
    * Update party's feed list.

package/src/pipeline/party-feed-provider.ts

@@ -7,9 +7,9 @@ import debug from 'debug';
 
 import { Event, synchronized } from '@dxos/async';
 import { Keyring, KeyType } from '@dxos/credentials';
-import { FeedSelector, FeedStoreIterator, MessageSelector, Timeframe } from '@dxos/echo-protocol';
+import { FeedSelector, FeedStoreIterator, MessageSelector } from '@dxos/echo-protocol';
 import { FeedDescriptor, FeedStore } from '@dxos/feed-store';
-import { PublicKey } from '@dxos/protocols';
+import { PublicKey, Timeframe } from '@dxos/protocols';
 import { ComplexMap } from '@dxos/util';
 
 import { MetadataStore } from './metadata-store';

package/src/pipeline/party-pipeline.test.ts

@@ -5,16 +5,16 @@
 import expect from 'expect';
 import { it as test } from 'mocha';
 
-import { promiseTimeout } from '@dxos/async';
+import { promiseTimeout, sleep } from '@dxos/async';
 import { createFeedAdmitMessage, createPartyGenesisMessage, Keyring, KeyType } from '@dxos/credentials';
 import { createId } from '@dxos/crypto';
 import { checkType } from '@dxos/debug';
-import { codec, FeedMessage, Timeframe } from '@dxos/echo-protocol';
+import { codec, FeedMessage } from '@dxos/echo-protocol';
 import { FeedStore } from '@dxos/feed-store';
 import { createTestProtocolPair } from '@dxos/mesh-protocol';
 import { ModelFactory } from '@dxos/model-factory';
 import { ObjectModel } from '@dxos/object-model';
-import { PublicKey } from '@dxos/protocols';
+import { PublicKey, Timeframe } from '@dxos/protocols';
 import { createStorage, StorageType } from '@dxos/random-access-multi-storage';
 import { afterTest } from '@dxos/testutils';
 
@@ -49,7 +49,7 @@ describe('PartyPipeline', () => {
     );
 
     const feed = await partyFeedProvider.createOrOpenWritableFeed();
-    await party.open({ feedHints: [feed.key] });
+    await party.open({ genesisFeedKey: feed.key });
     afterTest(async () => party.close());
 
     // PartyGenesis (self-signed by Party).
@@ -104,7 +104,7 @@ describe('PartyPipeline', () => {
     }
 
     await party.close();
-    await party.open({ feedHints: [feedKey] });
+    await party.open({ genesisFeedKey: feedKey });
 
     {
       await party.database.select().exec().update.waitFor(result => result.entities.length === 2);
@@ -132,7 +132,7 @@ describe('PartyPipeline', () => {
     expect(partyFeedProvider.getFeeds().find(k => k.key.equals(feedKey.publicKey))).toBeTruthy();
   });
 
-  test('opens feed from hints', async () => {
+  test('does not open unrelated feeds', async () => {
     const storage = createStorage('', StorageType.RAM);
     const feedStore = new FeedStore(storage.directory('feed'), { valueEncoding: codec });
     afterTest(async () => feedStore.close());
@@ -158,21 +158,19 @@ describe('PartyPipeline', () => {
       PublicKey.random()
     );
 
-    await partyFeedProvider.createOrOpenWritableFeed();
+    const writeFeed = await partyFeedProvider.createOrOpenWritableFeed();
 
-    const feedOpened = feedStore.feedOpenedEvent.waitForCount(1);
-
-    await party.open({ feedHints: [otherFeedKey] });
+    await party.open({ genesisFeedKey: writeFeed.key });
     afterTest(async () => party.close());
 
-    await feedOpened;
+    // Wait for events to be processed.
+    await sleep(5);
 
-    expect(partyFeedProvider.getFeeds().some(k => k.key.equals(otherFeedKey))).toEqual(true);
+    expect(partyFeedProvider.getFeeds().some(k => k.key.equals(otherFeedKey))).toEqual(false);
   });
 
   test('manually create item', async () => {
     const { party, partyFeedProvider } = await setup();
-    await party.open();
 
     const feed = await partyFeedProvider.createOrOpenWritableFeed();
 
@@ -193,7 +191,6 @@ describe('PartyPipeline', () => {
 
   test('admit a second feed to the party', async () => {
     const { party, keyring, partyKey, feedStore } = await setup();
-    await party.open();
 
     const feedKey = await keyring.createKeyRecord({ type: KeyType.FEED });
     const fullKey = keyring.getFullKey(feedKey.publicKey);
@@ -223,7 +220,6 @@ describe('PartyPipeline', () => {
 
   test('admit feed and then open it', async () => {
     const { party, keyring, partyKey, feedStore } = await setup();
-    await party.open();
 
     const feedKey = await keyring.createKeyRecord({ type: KeyType.FEED });
     const fullKey = keyring.getFullKey(feedKey.publicKey);
@@ -270,7 +266,7 @@ describe('PartyPipeline', () => {
     expect(timeframe.isEmpty()).toBeFalsy();
 
     await party.close();
-    await party.open({ feedHints: [feedKey], targetTimeframe: timeframe });
+    await party.open({ genesisFeedKey: feedKey, targetTimeframe: timeframe });
   });
 
   test('two instances replicating', async () => {
@@ -304,9 +300,7 @@ describe('PartyPipeline', () => {
       [peer1.partyKey]
     ));
 
-    await party2.open({
-      feedHints: [peer1.feedKey]
-    });
+    await party2.open({ genesisFeedKey: peer1.feedKey });
     afterTest(async () => party2.close());
 
     createTestProtocolPair(

package/src/pipeline/party-pipeline.ts

@@ -7,9 +7,11 @@ import assert from 'assert';
 import { synchronized } from '@dxos/async';
 import { KeyType, Message as HaloMessage } from '@dxos/credentials';
 import { timed } from '@dxos/debug';
-import { createFeedWriter, DatabaseSnapshot, FeedSelector, FeedWriter, PartyKey, PartySnapshot, Timeframe } from '@dxos/echo-protocol';
+import {
+  createFeedWriter, DatabaseSnapshot, FeedSelector, FeedWriter, PartyKey, PartySnapshot
+} from '@dxos/echo-protocol';
 import { ModelFactory } from '@dxos/model-factory';
-import { PublicKey } from '@dxos/protocols';
+import { PublicKey, Timeframe } from '@dxos/protocols';
 import { SubscriptionGroup } from '@dxos/util';
 
 import { createMessageSelector, PartyProcessor, PartyFeedProvider, FeedMuxer } from '.';
@@ -30,9 +32,9 @@ export interface PipelineOptions {
 
 export interface OpenOptions {
   /**
-   * Keys of initial feeds needed to bootstrap the party.
+   * Initial feed that contains PartyGenesis message and acts as the root for the feed DAG.
    */
-  feedHints?: PublicKey[]
+  genesisFeedKey: PublicKey
   /**
    * Timeframe to start processing feed messages from.
    */
@@ -123,9 +125,9 @@ export class PartyPipeline {
    */
   @synchronized
   @timed(1_000)
-  async open (options: OpenOptions = {}) {
+  async open (options: OpenOptions) {
     const {
-      feedHints = [],
+      genesisFeedKey,
       initialTimeframe,
       targetTimeframe
     } = options;
@@ -147,17 +149,16 @@ export class PartyPipeline {
       void this._feedProvider.createOrOpenReadOnlyFeed(feed);
     }));
 
-    if (feedHints.length > 0) {
-      await this._partyProcessor.takeHints(feedHints.map(publicKey => ({ publicKey, type: KeyType.FEED })));
-    }
+    // TODO(dmaretskyi): We still need to hint at the genesis feed for some reason, not doing this breaks invitation tests.
+    await this._partyProcessor.takeHints([{ type: KeyType.FEED, publicKey: genesisFeedKey }]);
 
     //
     // Pipeline
     //
 
     const iterator = await this._feedProvider.createIterator(
-      createMessageSelector(this._partyProcessor, this._timeframeClock),
-      createFeedSelector(this._partyProcessor, feedHints),
+      createMessageSelector(this._timeframeClock),
+      createFeedSelector(this._partyProcessor, genesisFeedKey),
       initialTimeframe
     );
 
@@ -256,4 +257,4 @@ export class PartyPipeline {
   }
 }
 
-const createFeedSelector = (partyProcessor: PartyProcessor, hints: PublicKey[]): FeedSelector => feed => hints.some(hint => hint.equals(feed.key)) || partyProcessor.isFeedAdmitted(feed.key);
+const createFeedSelector = (partyProcessor: PartyProcessor, genesisFeed: PublicKey): FeedSelector => feed => genesisFeed.equals(feed.key) || partyProcessor.isFeedAdmitted(feed.key);