@dxos/echo-db 2.33.5-dev.ea3876ba → 2.33.5-dev.fee8f5fe

package/src/pipeline/party-feed-provider.ts

@@ -8,7 +8,7 @@ import debug from 'debug';
 import { Event, synchronized } from '@dxos/async';
 import { Keyring, KeyType } from '@dxos/credentials';
 import { PublicKey } from '@dxos/crypto';
-import { FeedStoreIterator, MessageSelector, Timeframe } from '@dxos/echo-protocol';
+import { FeedSelector, FeedStoreIterator, MessageSelector, Timeframe } from '@dxos/echo-protocol';
 import { FeedDescriptor, FeedStore } from '@dxos/feed-store';
 import { ComplexMap } from '@dxos/util';
 
@@ -33,19 +33,6 @@ export class PartyFeedProvider {
     return Array.from(this._feeds.values());
   }
 
-  @synchronized
-  async openKnownFeeds () {
-    for (const feedKey of this._metadataStore.getParty(this._partyKey)?.feedKeys ?? []) {
-      if (!this._feeds.has(feedKey)) {
-        const fullKey = this._keyring.getFullKey(feedKey);
-        const feed = fullKey?.secretKey
-          ? await this._feedStore.openReadWriteFeed(fullKey.publicKey, fullKey.secretKey)
-          : await this._feedStore.openReadOnlyFeed(feedKey);
-        this._trackFeed(feed);
-      }
-    }
-  }
-
   @synchronized
   async createOrOpenWritableFeed () {
     const partyMetadata = this._metadataStore.getParty(this._partyKey);
@@ -103,8 +90,8 @@ export class PartyFeedProvider {
     return feed;
   }
 
-  async createIterator (messageSelector: MessageSelector, initialTimeframe?: Timeframe) {
-    const iterator = new FeedStoreIterator(() => true, messageSelector, initialTimeframe ?? new Timeframe());
+  async createIterator (messageSelector: MessageSelector, feedSelector: FeedSelector, initialTimeframe?: Timeframe) {
+    const iterator = new FeedStoreIterator(feedSelector, messageSelector, initialTimeframe ?? new Timeframe());
     for (const feed of this._feeds.values()) {
       iterator.addFeedDescriptor(feed);
     }

package/src/pipeline/{party-core.test.ts → party-pipeline.test.ts}

@@ -8,7 +8,8 @@ import { it as test } from 'mocha';
 import { promiseTimeout } from '@dxos/async';
 import { createFeedAdmitMessage, createPartyGenesisMessage, Keyring, KeyType } from '@dxos/credentials';
 import { createId, PublicKey } from '@dxos/crypto';
-import { codec, Timeframe } from '@dxos/echo-protocol';
+import { checkType } from '@dxos/debug';
+import { codec, FeedMessage, Timeframe } from '@dxos/echo-protocol';
 import { FeedStore } from '@dxos/feed-store';
 import { createTestProtocolPair } from '@dxos/mesh-protocol';
 import { ModelFactory } from '@dxos/model-factory';
@@ -16,28 +17,29 @@ import { ObjectModel } from '@dxos/object-model';
 import { createStorage, StorageType } from '@dxos/random-access-multi-storage';
 import { afterTest } from '@dxos/testutils';
 
-import { MetadataStore, PartyFeedProvider, ReplicatorProtocolPluginFactory } from '../pipeline';
+import { MetadataStore, PartyFeedProvider } from '.';
+import { createReplicatorPlugin } from '../protocol/replicator-plugin';
 import { SnapshotStore } from '../snapshots';
-import { PartyCore } from './party-core';
+import { PartyPipeline } from './party-pipeline';
 
-describe('PartyCore', () => {
+describe('PartyPipeline', () => {
   const setup = async () => {
     const storage = createStorage('', StorageType.RAM);
-    const feedStore = new FeedStore(storage, { valueEncoding: codec });
+    const feedStore = new FeedStore(storage.directory('feed'), { valueEncoding: codec });
     afterTest(async () => feedStore.close());
 
     const keyring = new Keyring();
 
-    const metadataStore = new MetadataStore(createStorage('metadata', StorageType.RAM));
+    const metadataStore = new MetadataStore(storage.directory('metadata'));
 
     const modelFactory = new ModelFactory().registerModel(ObjectModel);
-    const snapshotStore = new SnapshotStore(createStorage('snapshots', StorageType.RAM));
+    const snapshotStore = new SnapshotStore(storage.directory('snapshots'));
 
     const partyKey = await keyring.createKeyRecord({ type: KeyType.PARTY });
 
     const partyFeedProvider = new PartyFeedProvider(metadataStore, keyring, feedStore, partyKey.publicKey);
 
-    const party = new PartyCore(
+    const party = new PartyPipeline(
       partyKey.publicKey,
       partyFeedProvider,
       modelFactory,
@@ -46,11 +48,11 @@ describe('PartyCore', () => {
     );
 
     const feed = await partyFeedProvider.createOrOpenWritableFeed();
-    await party.open();
+    await party.open({ feedHints: [feed.key] });
     afterTest(async () => party.close());
 
     // PartyGenesis (self-signed by Party).
-    await party.processor.writeHaloMessage(createPartyGenesisMessage(
+    await party.credentialsWriter.write(createPartyGenesisMessage(
       keyring,
       partyKey,
       feed.key,
@@ -58,7 +60,7 @@ describe('PartyCore', () => {
     );
 
     // FeedAdmit (signed by the Device KeyChain).
-    await party.processor.writeHaloMessage(createFeedAdmitMessage(
+    await party.credentialsWriter.write(createFeedAdmitMessage(
       keyring,
       partyKey.publicKey,
       feed.key,
@@ -86,7 +88,7 @@ describe('PartyCore', () => {
   });
 
   test('create item with parent and then reload', async () => {
-    const { party } = await setup();
+    const { party, feedKey } = await setup();
 
     {
       const parent = await party.database.createItem({ model: ObjectModel, type: 'parent' });
@@ -101,7 +103,7 @@ describe('PartyCore', () => {
     }
 
     await party.close();
-    await party.open();
+    await party.open({ feedHints: [feedKey] });
 
     {
       await party.database.select().exec().update.waitFor(result => result.entities.length === 2);
@@ -119,7 +121,7 @@ describe('PartyCore', () => {
     const feedKey = await keyring.createKeyRecord({ type: KeyType.FEED });
 
     const eventFired = feedStore.feedOpenedEvent.waitForCount(1);
-    await party.processor.writeHaloMessage(createFeedAdmitMessage(
+    await party.credentialsWriter.write(createFeedAdmitMessage(
       keyring,
       party.key,
       feedKey.publicKey,
@@ -131,15 +133,15 @@ describe('PartyCore', () => {
 
   test('opens feed from hints', async () => {
     const storage = createStorage('', StorageType.RAM);
-    const feedStore = new FeedStore(storage, { valueEncoding: codec });
+    const feedStore = new FeedStore(storage.directory('feed'), { valueEncoding: codec });
     afterTest(async () => feedStore.close());
 
     const keyring = new Keyring();
 
-    const metadataStore = new MetadataStore(createStorage('metadata', StorageType.RAM));
+    const metadataStore = new MetadataStore(storage.directory('metadata'));
 
     const modelFactory = new ModelFactory().registerModel(ObjectModel);
-    const snapshotStore = new SnapshotStore(createStorage('snapshots', StorageType.RAM));
+    const snapshotStore = new SnapshotStore(storage.directory('snapshots'));
 
     const partyKey = await keyring.createKeyRecord({ type: KeyType.PARTY });
 
@@ -147,7 +149,7 @@ describe('PartyCore', () => {
 
     const otherFeedKey = PublicKey.random();
 
-    const party = new PartyCore(
+    const party = new PartyPipeline(
       partyKey.publicKey,
       partyFeedProvider,
       modelFactory,
@@ -159,7 +161,7 @@ describe('PartyCore', () => {
 
     const feedOpened = feedStore.feedOpenedEvent.waitForCount(1);
 
-    await party.open([{ type: KeyType.FEED, publicKey: otherFeedKey }]);
+    await party.open({ feedHints: [otherFeedKey] });
     afterTest(async () => party.close());
 
     await feedOpened;
@@ -174,16 +176,16 @@ describe('PartyCore', () => {
     const feed = await partyFeedProvider.createOrOpenWritableFeed();
 
     const itemId = createId();
-    await feed.feed.append({
+    await feed.feed.append(checkType<FeedMessage>({
+      timeframe: new Timeframe(),
       echo: {
         itemId,
         genesis: {
           itemType: 'dxos:example',
           modelType: ObjectModel.meta.type
-        },
-        timeframe: new Timeframe()
+        }
       }
-    });
+    }));
 
     await promiseTimeout(party.database.waitForItem({ id: itemId }), 1000, new Error('timeout'));
   });
@@ -196,7 +198,7 @@ describe('PartyCore', () => {
     const fullKey = keyring.getFullKey(feedKey.publicKey);
     const feed2 = await feedStore.openReadWriteFeed(fullKey!.publicKey, fullKey!.secretKey!);
 
-    await party.processor.writeHaloMessage(createFeedAdmitMessage(
+    await party.credentialsWriter.write(createFeedAdmitMessage(
       keyring,
       party.key,
       feed2.key,
@@ -204,16 +206,16 @@ describe('PartyCore', () => {
     ));
 
     const itemId = createId();
-    await feed2.append({
+    await feed2.append(checkType<FeedMessage>({
+      timeframe: new Timeframe(),
       echo: {
         itemId,
         genesis: {
           itemType: 'dxos:example',
           modelType: ObjectModel.meta.type
-        },
-        timeframe: new Timeframe()
+        }
       }
-    });
+    }));
 
     await promiseTimeout(party.database.waitForItem({ id: itemId }), 1000, new Error('timeout'));
   });
@@ -225,7 +227,7 @@ describe('PartyCore', () => {
     const feedKey = await keyring.createKeyRecord({ type: KeyType.FEED });
     const fullKey = keyring.getFullKey(feedKey.publicKey);
 
-    await party.processor.writeHaloMessage(createFeedAdmitMessage(
+    await party.credentialsWriter.write(createFeedAdmitMessage(
       keyring,
       party.key,
       feedKey.publicKey,
@@ -234,35 +236,57 @@ describe('PartyCore', () => {
 
     const feed2 = await feedStore.openReadWriteFeed(fullKey!.publicKey, fullKey!.secretKey!);
     const itemId = createId();
-    await feed2.append({
+    await feed2.append(checkType<FeedMessage>({
+      timeframe: new Timeframe(),
       echo: {
         itemId,
         genesis: {
           itemType: 'dxos:example',
           modelType: ObjectModel.meta.type
-        },
-        timeframe: new Timeframe()
+        }
       }
-    });
+    }));
 
     await promiseTimeout(party.database.waitForItem({ id: itemId }), 1000, new Error('timeout'));
   });
 
+  test('wait to reach specific timeframe', async () => {
+    const { party, feedKey } = await setup();
+
+    {
+      const parent = await party.database.createItem({ model: ObjectModel, type: 'parent' });
+      const child = await party.database.createItem({
+        model: ObjectModel,
+        parent: parent.id,
+        type: 'child'
+      });
+
+      expect(child.parent).toEqual(parent);
+      expect(parent.children).toContain(child);
+    }
+
+    const timeframe = party.timeframe;
+    expect(timeframe.isEmpty()).toBeFalsy();
+
+    await party.close();
+    await party.open({ feedHints: [feedKey], targetTimeframe: timeframe });
+  });
+
   test('two instances replicating', async () => {
     const peer1 = await setup();
 
     const storage = createStorage('', StorageType.RAM);
-    const feedStore = new FeedStore(storage, { valueEncoding: codec });
+    const feedStore = new FeedStore(storage.directory('feed'), { valueEncoding: codec });
     afterTest(async () => feedStore.close());
 
-    const metadataStore = new MetadataStore(createStorage('metadata', StorageType.RAM));
+    const metadataStore = new MetadataStore(storage.directory('metadata'));
 
     const modelFactory = new ModelFactory().registerModel(ObjectModel);
-    const snapshotStore = new SnapshotStore(createStorage('snapshots', StorageType.RAM));
+    const snapshotStore = new SnapshotStore(storage.directory('snapshots'));
 
     const partyFeedProvider = new PartyFeedProvider(metadataStore, peer1.keyring, feedStore, peer1.party.key);
 
-    const party2 = new PartyCore(
+    const party2 = new PartyPipeline(
       peer1.party.key,
       partyFeedProvider,
       modelFactory,
@@ -272,26 +296,21 @@ describe('PartyCore', () => {
 
     const feed2 = await partyFeedProvider.createOrOpenWritableFeed();
 
-    await peer1.party.processor.writeHaloMessage(createFeedAdmitMessage(
+    await peer1.party.credentialsWriter.write(createFeedAdmitMessage(
       peer1.keyring,
       peer1.party.key,
       feed2.key,
       [peer1.partyKey]
     ));
 
-    await party2.open([{
-      publicKey: peer1.feedKey,
-      type: KeyType.FEED
-    }]);
+    await party2.open({
+      feedHints: [peer1.feedKey]
+    });
     afterTest(async () => party2.close());
 
     createTestProtocolPair(
-      new ReplicatorProtocolPluginFactory(
-        peer1.partyFeedProvider
-      ).createPlugins().map(r => r.createExtension()),
-      new ReplicatorProtocolPluginFactory(
-        partyFeedProvider
-      ).createPlugins().map(r => r.createExtension())
+      [createReplicatorPlugin(peer1.partyFeedProvider).createExtension()],
+      [createReplicatorPlugin(partyFeedProvider).createExtension()]
     );
 
     const item1 = await peer1.party.database.createItem();

package/src/pipeline/{party-core.ts → party-pipeline.ts}

@@ -5,21 +5,20 @@
 import assert from 'assert';
 
 import { synchronized } from '@dxos/async';
-import { KeyHint, KeyType } from '@dxos/credentials';
+import { KeyType, Message as HaloMessage } from '@dxos/credentials';
 import { PublicKey } from '@dxos/crypto';
 import { timed } from '@dxos/debug';
-import { createFeedWriter, DatabaseSnapshot, PartyKey, PartySnapshot, Timeframe } from '@dxos/echo-protocol';
+import { createFeedWriter, DatabaseSnapshot, FeedSelector, FeedWriter, PartyKey, PartySnapshot, Timeframe } from '@dxos/echo-protocol';
 import { ModelFactory } from '@dxos/model-factory';
 import { SubscriptionGroup } from '@dxos/util';
 
-import { Database } from '../api';
-import { FeedDatabaseBackend, TimeframeClock } from '../database';
-import { createMessageSelector, PartyProcessor, PartyFeedProvider, Pipeline } from '../pipeline';
+import { createMessageSelector, PartyProcessor, PartyFeedProvider, FeedMuxer } from '.';
+import { Database, FeedDatabaseBackend, TimeframeClock } from '../packlets/database';
 import { createAutomaticSnapshots, SnapshotStore } from '../snapshots';
 
 const DEFAULT_SNAPSHOT_INTERVAL = 100; // Every 100 messages.
 
-export interface PartyOptions {
+export interface PipelineOptions {
   readLogger?: (msg: any) => void;
   writeLogger?: (msg: any) => void;
   readOnly?: boolean;
@@ -29,6 +28,22 @@ export interface PartyOptions {
   snapshotInterval?: number;
 }
 
+export interface OpenOptions {
+  /**
+   * Keys of initial feeds needed to bootstrap the party.
+   */
+  feedHints?: PublicKey[]
+  /**
+   * Timeframe to start processing feed messages from.
+   */
+  initialTimeframe?: Timeframe
+  /**
+   * Timeframe which must be reached until further processing.
+   * PartyCore.open will block until this timeframe is reached.
+   */
+  targetTimeframe?: Timeframe
+}
+
 /**
  * Encapsulates core components needed by a party:
  * - ECHO database with item-manager & item-demuxer.
@@ -37,7 +52,7 @@ export interface PartyOptions {
  *
  * The core class also handles the combined ECHO and HALO state snapshots.
  */
-export class PartyCore {
+export class PartyPipeline {
   /**
    * Snapshot to be restored from when party.open() is called.
    */
@@ -46,7 +61,7 @@ export class PartyCore {
   private readonly _subscriptions = new SubscriptionGroup();
 
   private _database?: Database;
-  private _pipeline?: Pipeline;
+  private _pipeline?: FeedMuxer;
   private _partyProcessor?: PartyProcessor;
   private _timeframeClock?: TimeframeClock;
 
@@ -56,8 +71,7 @@ export class PartyCore {
     private readonly _modelFactory: ModelFactory,
     private readonly _snapshotStore: SnapshotStore,
     private readonly _memberKey: PublicKey,
-    private readonly _initialTimeframe?: Timeframe,
-    private readonly _options: PartyOptions = {}
+    private readonly _options: PipelineOptions = {}
   ) { }
 
   get key (): PartyKey {
@@ -99,20 +113,29 @@ export class PartyCore {
     return feed;
   }
 
+  get credentialsWriter (): FeedWriter<HaloMessage> {
+    assert(this._pipeline?.outboundHaloStream, 'Party not open');
+    return this._pipeline?.outboundHaloStream;
+  }
+
   /**
    * Opens the pipeline and connects the streams.
    */
   @synchronized
   @timed(1_000)
-  async open (keyHints: KeyHint[] = []) {
+  async open (options: OpenOptions = {}) {
+    const {
+      feedHints = [],
+      initialTimeframe,
+      targetTimeframe
+    } = options;
+
     if (this.isOpen) {
       return this;
     }
 
-    this._timeframeClock = new TimeframeClock(this._initialTimeframe);
+    this._timeframeClock = new TimeframeClock(initialTimeframe);
 
-    // Open all feeds known from metadata and open or create a writable feed to the party.
-    await this._feedProvider.openKnownFeeds();
     const writableFeed = await this._feedProvider.createOrOpenWritableFeed();
 
     if (!this._partyProcessor) {
@@ -124,12 +147,8 @@ export class PartyCore {
       void this._feedProvider.createOrOpenReadOnlyFeed(feed);
     }));
 
-    // Hint at our own writable feed.
-    // TODO(dmaretskyi): Does not seem like it should be required, but without it replication between devices (B -> A) breaks.
-    await this._partyProcessor.takeHints([{ type: KeyType.FEED, publicKey: writableFeed.key }]);
-
-    if (keyHints.length > 0) {
-      await this._partyProcessor.takeHints(keyHints);
+    if (feedHints.length > 0) {
+      await this._partyProcessor.takeHints(feedHints.map(publicKey => ({ publicKey, type: KeyType.FEED })));
     }
 
     //
@@ -138,31 +157,34 @@ export class PartyCore {
 
     const iterator = await this._feedProvider.createIterator(
       createMessageSelector(this._partyProcessor, this._timeframeClock),
-      this._initialTimeframe
+      createFeedSelector(this._partyProcessor, feedHints),
+      initialTimeframe
     );
 
-    this._pipeline = new Pipeline(
-      this._partyProcessor, iterator, this._timeframeClock, createFeedWriter(writableFeed.feed), this._options);
+    this._pipeline = new FeedMuxer(
+      this._partyProcessor,
 
-    // TODO(burdon): Support read-only parties.
-    const [readStream, writeStream] = await this._pipeline.open();
-
-    // Must happen after open.
-    if (this._pipeline.outboundHaloStream) {
-      this._partyProcessor.setOutboundStream(this._pipeline.outboundHaloStream);
-    }
+      iterator,
+      this._timeframeClock,
+      createFeedWriter(writableFeed.feed),
+      this._options
+    );
 
     //
     // Database
     //
 
+    const databaseBackend = new FeedDatabaseBackend(this._pipeline.outboundEchoStream, this._databaseSnapshot, { snapshots: true });
     this._database = new Database(
       this._modelFactory,
-      new FeedDatabaseBackend(readStream, writeStream, this._databaseSnapshot, { snapshots: true }),
+      databaseBackend,
       this._memberKey
     );
 
+    // Open pipeline and connect it to the database.
     await this._database.initialize();
+    this._pipeline.setEchoProcessor(databaseBackend.echoProcessor);
+    await this._pipeline.open();
 
     // TODO(burdon): Propagate errors.
     this._subscriptions.push(this._pipeline.errors.on(err => console.error(err)));
@@ -176,6 +198,10 @@ export class PartyCore {
       );
     }
 
+    if (targetTimeframe) {
+      await this._timeframeClock.waitUntilReached(targetTimeframe);
+    }
+
     return this;
   }
 
@@ -229,3 +255,5 @@ export class PartyCore {
     this._databaseSnapshot = snapshot.database;
   }
 }
+
+const createFeedSelector = (partyProcessor: PartyProcessor, hints: PublicKey[]): FeedSelector => feed => hints.some(hint => hint.equals(feed.key)) || partyProcessor.isFeedAdmitted(feed.key);

package/src/pipeline/party-processor.ts

@@ -12,22 +12,40 @@ import {
   PartyState,
   Message as HaloMessage,
   IdentityEventType,
-  PartyEventType
+  PartyEventType,
+  SignedMessage
 } from '@dxos/credentials';
 import { PublicKey } from '@dxos/crypto';
-import { FeedKey, FeedWriter, IHaloStream, PartyKey, HaloStateSnapshot, WriteReceipt } from '@dxos/echo-protocol';
+import { FeedKey, IHaloStream, PartyKey, HaloStateSnapshot } from '@dxos/echo-protocol';
 import { jsonReplacer } from '@dxos/util';
 
 const log = debug('dxos:echo-db:party-processor');
 
+export interface CredentialProcessor {
+  processMessage (message: IHaloStream): Promise<void>
+}
+
+export interface PartyStateProvider {
+  partyKey: PublicKey
+
+  /**
+   * Whether PartyGenesis was already processed.
+   */
+  genesisRequired: boolean
+  memberKeys: PublicKey[]
+  feedKeys: PublicKey[]
+  getFeedOwningMember (feedKey: FeedKey): PublicKey | undefined
+  isFeedAdmitted (feedKey: FeedKey): boolean
+
+  getOfflineInvitation (invitationID: Buffer): SignedMessage | undefined
+}
+
 /**
  * TODO(burdon): Wrapper/Bridge between HALO APIs.
  */
-export class PartyProcessor {
+export class PartyProcessor implements CredentialProcessor, PartyStateProvider {
   private readonly _state: PartyState;
 
-  private _outboundHaloStream: FeedWriter<HaloMessage> | undefined;
-
   readonly feedAdded = new Event<FeedKey>()
 
   public readonly keyOrInfoAdded = new Event<PublicKey>();
@@ -124,16 +142,6 @@ export class PartyProcessor {
     await this._state.processMessages([data]);
   }
 
-  setOutboundStream (stream: FeedWriter<HaloMessage>) {
-    this._outboundHaloStream = stream;
-  }
-
-  async writeHaloMessage (message: HaloMessage): Promise<WriteReceipt> {
-    assert(this._outboundHaloStream, 'Party is closed or read-only');
-    // TODO(marik-d): Wait for the message to be processed?
-    return this._outboundHaloStream.write(message);
-  }
-
   makeSnapshot (): HaloStateSnapshot {
     return {
       messages: this._haloMessages

package/src/protocol/auth-plugin.ts

@@ -9,6 +9,4 @@ import { Replicator } from '@dxos/protocol-plugin-replicator';
 /**
  * Creates authenticator network-protocol plugin that guards access to the replicator.
  */
-export function createAuthPlugin (authenticator: Authenticator, peerId: PublicKey) {
-  return new AuthPlugin(peerId.asBuffer(), authenticator, [Replicator.extension]);
-}
+export const createAuthPlugin = (authenticator: Authenticator, peerId: PublicKey) => new AuthPlugin(peerId.asBuffer(), authenticator, [Replicator.extension]);

package/src/protocol/authenticator.test.ts

@@ -6,7 +6,6 @@ import expect from 'expect';
 import { it as test } from 'mocha';
 
 import { createAuthMessage, createKeyAdmitMessage, createPartyGenesisMessage, Keyring, KeyType } from '@dxos/credentials';
-import { MockFeedWriter } from '@dxos/echo-protocol';
 
 import { PartyProcessor } from '../pipeline';
 import { createAuthenticator } from './authenticator';
@@ -23,8 +22,6 @@ describe('authenticator', () => {
     const signer = CredentialsSigner.createDirectDeviceSigner(keyring);
 
     const partyProcessor = new PartyProcessor(partyKey.publicKey);
-    const feed = new MockFeedWriter();
-    partyProcessor.setOutboundStream(feed);
     await partyProcessor.processMessage({
       data: createPartyGenesisMessage(
         keyring,
@@ -53,7 +50,7 @@ describe('authenticator', () => {
       meta: {} as any
     });
 
-    const authenticator = createAuthenticator(partyProcessor, signer);
+    const authenticator = createAuthenticator(partyProcessor, signer, null as any);
     const credential = createAuthMessage(
       keyring,
       partyKey.publicKey,