@dxos/echo-db 2.33.5-dev.1fc05a51 → 2.33.5-dev.39ec2a53

package/src/{api → packlets/database}/database.ts

@@ -10,9 +10,11 @@ import { ItemID, ItemType } from '@dxos/echo-protocol';
 import { Model, ModelConstructor, ModelFactory, validateModelClass } from '@dxos/model-factory';
 import { ObjectModel } from '@dxos/object-model';
 
-import { DatabaseBackend, DataServiceHost, ItemManager } from '../database';
+import { DataServiceHost } from './data-service-host';
+import { DatabaseBackend } from './database-backend';
 import { Entity } from './entity';
 import { Item } from './item';
+import { ItemManager } from './item-manager';
 import { Link } from './link';
 import { RootFilter, Selection, createSelection } from './selection';
 

package/src/{api → packlets/database}/entity.ts

@@ -7,7 +7,7 @@ import { ItemID, ItemType } from '@dxos/echo-protocol';
 import { Model, ModelMeta, StateManager } from '@dxos/model-factory';
 import { SubscriptionGroup } from '@dxos/util';
 
-import { ItemManager } from '../database';
+import { ItemManager } from './item-manager';
 
 /**
  * Base class for all ECHO entitities.

package/src/{database → packlets/database}/index.ts

@@ -10,3 +10,8 @@ export * from './item-demuxer';
 export * from './item-manager';
 export * from './testing';
 export * from './timeframe-clock';
+export * from './item';
+export * from './link';
+export * from './entity';
+export * from './selection';
+export * from './database';

package/src/{database → packlets/database}/item-demuxer.test.ts

@@ -9,12 +9,11 @@ import { it as test } from 'mocha';
 import { latch } from '@dxos/async';
 import { createId, PublicKey } from '@dxos/crypto';
 import { checkType } from '@dxos/debug';
-import { createMockFeedWriterFromStream, EchoEnvelope, IEchoStream, MockFeedWriter, Timeframe } from '@dxos/echo-protocol';
-import { createTransform } from '@dxos/feed-store';
+import { EchoEnvelope, MockFeedWriter, Timeframe } from '@dxos/echo-protocol';
 import { ModelFactory, TestModel } from '@dxos/model-factory';
 import { ObjectModel } from '@dxos/object-model';
 
-import { Item } from '../api';
+import { Item } from './item';
 import { ItemDemuxer } from './item-demuxer';
 import { ItemManager } from './item-manager';
 
@@ -32,7 +31,7 @@ describe('Item demuxer', () => {
     const itemDemuxer = new ItemDemuxer(itemManager, modelFactory);
 
     const inboundStream = itemDemuxer.open();
-    feedWriter.written.on(([msg, meta]) => inboundStream.write({
+    feedWriter.written.on(([msg, meta]) => inboundStream({
       data: msg,
       meta: { ...meta, memberKey }
     } as any));
@@ -96,28 +95,31 @@ describe('Item demuxer', () => {
     const modelFactory = new ModelFactory()
       .registerModel(ObjectModel);
 
-    const writeStream = createTransform<EchoEnvelope, IEchoStream>(
-      async (message: EchoEnvelope): Promise<IEchoStream> => ({
-        meta: {
-          feedKey: PublicKey.random(),
-          memberKey: PublicKey.random(),
-          seq: 0,
-          timeframe: new Timeframe()
-        },
-        data: message
-      })
-    );
-    const itemManager = new ItemManager(modelFactory, PublicKey.random(), createMockFeedWriterFromStream(writeStream));
+    const itemManager = new ItemManager(modelFactory, PublicKey.random(), {
+      write: async (message) => {
+        void processEchoMessage(message);
+        return { feedKey: PublicKey.random(), seq: 0 };
+      }
+    });
     const itemDemuxer = new ItemDemuxer(itemManager, modelFactory);
-    writeStream.pipe(itemDemuxer.open());
+    const processor = itemDemuxer.open();
+    const processEchoMessage = (message: EchoEnvelope) => processor({
+      meta: {
+        feedKey: PublicKey.random(),
+        memberKey: PublicKey.random(),
+        seq: 0,
+        timeframe: new Timeframe()
+      },
+      data: message
+    });
 
-    writeStream.write(checkType<EchoEnvelope>({
+    void processEchoMessage(checkType<EchoEnvelope>({
       itemId: 'foo',
       genesis: {
         modelType: TestModel.meta.type
       }
     }));
-    writeStream.write(checkType<EchoEnvelope>({
+    void processEchoMessage(checkType<EchoEnvelope>({
       itemId: 'bar',
       genesis: {
         modelType: ObjectModel.meta.type

package/src/{database → packlets/database}/item-demuxer.ts

@@ -8,11 +8,12 @@ import debug from 'debug';
 import { Event } from '@dxos/async';
 import { failUndefined } from '@dxos/debug';
 import { DatabaseSnapshot, IEchoStream, ItemID, ItemSnapshot, LinkSnapshot } from '@dxos/echo-protocol';
-import { createWritable } from '@dxos/feed-store';
 import { Model, ModelFactory, ModelMessage } from '@dxos/model-factory';
 
-import { Entity, Item, Link } from '../api';
+import { Entity } from './entity';
+import { Item } from './item';
 import { ItemManager, ModelConstructionOptions } from './item-manager';
+import { Link } from './link';
 
 const log = debug('dxos:echo-db:item-demuxer');
 
@@ -20,6 +21,8 @@ export interface ItemDemuxerOptions {
   snapshots?: boolean
 }
 
+export type EchoProcessor = (message: IEchoStream) => Promise<void>
+
 /**
  * Creates a stream that consumes `IEchoStream` messages and routes them to the associated items.
  * @param itemManager
@@ -33,7 +36,7 @@ export class ItemDemuxer {
     private readonly _options: ItemDemuxerOptions = {}
   ) {}
 
-  open (): NodeJS.WritableStream {
+  open (): EchoProcessor {
     this._modelFactory.registered.on(async model => {
       for (const item of this._itemManager.getUninitializedEntities()) {
         if (item._stateManager.modelType === model.meta.type) {
@@ -44,7 +47,7 @@ export class ItemDemuxer {
 
     // TODO(burdon): Factor out.
     // TODO(burdon): Should this implement some "back-pressure" (hints) to the PartyProcessor?
-    return createWritable<IEchoStream>(async (message: IEchoStream) => {
+    return async (message: IEchoStream) => {
       const { data: { itemId, genesis, itemMutation, mutation, snapshot }, meta } = message;
       assert(itemId);
 
@@ -109,7 +112,7 @@ export class ItemDemuxer {
       }
 
       this.mutation.emit(message);
-    });
+    };
   }
 
   createSnapshot (): DatabaseSnapshot {

package/src/{database → packlets/database}/item-manager.ts

@@ -11,8 +11,10 @@ import { timed } from '@dxos/debug';
 import { EchoEnvelope, FeedWriter, ItemID, ItemType, mapFeedWriter, ModelSnapshot } from '@dxos/echo-protocol';
 import { Model, ModelFactory, ModelMessage, ModelType, StateManager } from '@dxos/model-factory';
 
-import { Entity, Item, Link } from '../api';
 import { UnknownModelError } from '../errors';
+import { Entity } from './entity';
+import { Item } from './item';
+import { Link } from './link';
 
 const log = debug('dxos:echo-db:item-manager');
 

package/src/{api → packlets/database}/item.ts

@@ -7,10 +7,10 @@ import debug from 'debug';
 import { EchoEnvelope, ItemID, ItemMutation, ItemType, FeedWriter } from '@dxos/echo-protocol';
 import { Model, StateManager } from '@dxos/model-factory';
 
-import { ItemManager } from '../database';
 import { Entity } from './entity';
+import { ItemManager } from './item-manager';
 import type { Link } from './link';
-import { Selection, createItemSelection } from './selection';
+import { createItemSelection, Selection } from './selection';
 
 const log = debug('dxos:echo-db:item');
 

package/src/{api → packlets/database}/link.ts

@@ -7,9 +7,9 @@ import assert from 'assert';
 import { ItemID, ItemType } from '@dxos/echo-protocol';
 import { Model, StateManager } from '@dxos/model-factory';
 
-import { ItemManager } from '../database';
 import { Entity } from './entity';
 import { Item } from './item';
+import { ItemManager } from './item-manager';
 
 export interface LinkData {
   sourceId: ItemID

package/src/{database → packlets/database}/testing.ts

@@ -2,25 +2,22 @@
 // Copyright 2021 DXOS.org
 //
 
-import { Readable } from 'stream';
-
 import { PublicKey } from '@dxos/crypto';
 import { EchoEnvelope, MockFeedWriter, Timeframe } from '@dxos/echo-protocol';
 import { ModelFactory } from '@dxos/model-factory';
 
-import { Database } from '../api';
 import { DataServiceHost } from './data-service-host';
 import { DataServiceRouter } from './data-service-router';
+import { Database } from './database';
 import { FeedDatabaseBackend, RemoteDatabaseBackend } from './database-backend';
 
 export const createInMemoryDatabase = async (modelFactory: ModelFactory) => {
   const feed = new MockFeedWriter<EchoEnvelope>();
-  const inboundStream = new Readable({ read: () => {}, objectMode: true });
-  feed.written.on(([data, meta]) => inboundStream.push({ data, meta: { ...meta, memberKey: PublicKey.random(), timeframe: new Timeframe([[meta.feedKey, meta.seq]]) } }));
-
+  const backend = new FeedDatabaseBackend(feed, undefined, { snapshots: true });
+  feed.written.on(([data, meta]) => backend.echoProcessor({ data, meta: { ...meta, memberKey: PublicKey.random(), timeframe: new Timeframe([[meta.feedKey, meta.seq]]) } }));
   const database = new Database(
     modelFactory,
-    new FeedDatabaseBackend(inboundStream, feed, undefined, { snapshots: true }),
+    backend,
     PublicKey.random()
   );
 

package/src/{database → packlets/database}/timeframe-clock.ts

@@ -28,4 +28,8 @@ export class TimeframeClock {
     const gaps = Timeframe.dependencies(timeframe, this._timeframe);
     return !gaps.isEmpty();
   }
+
+  async waitUntilReached (target: Timeframe) {
+    await this.update.waitForCondition(() => Timeframe.dependencies(target, this._timeframe).isEmpty());
+  }
 }

package/src/parties/data-party.test.ts

@@ -5,7 +5,7 @@
 import expect from 'expect';
 import { it as test } from 'mocha';
 
-import { createKeyAdmitMessage, createPartyGenesisMessage, defaultSecretProvider, KeyHint, Keyring, KeyType, codec as haloCodec } from '@dxos/credentials';
+import { createKeyAdmitMessage, createPartyGenesisMessage, defaultSecretProvider, Keyring, KeyType, codec as haloCodec } from '@dxos/credentials';
 import { PublicKey } from '@dxos/crypto';
 import { codec } from '@dxos/echo-protocol';
 import { FeedStore } from '@dxos/feed-store';
@@ -22,7 +22,7 @@ import { SnapshotStore } from '../snapshots';
 import { DataParty } from './data-party';
 
 describe('DataParty', () => {
-  const createParty = async (identity: IdentityCredentials, partyKey: PublicKey, hints: KeyHint[]) => {
+  const createParty = async (identity: IdentityCredentials, partyKey: PublicKey, feedHints: PublicKey[]) => {
 
     const storage = createStorage('', StorageType.RAM);
     const snapshotStore = new SnapshotStore(storage.directory('snapshots'));
@@ -31,17 +31,20 @@ describe('DataParty', () => {
     const modelFactory = new ModelFactory().registerModel(ObjectModel);
     const networkManager = new NetworkManager();
     const partyFeedProvider = new PartyFeedProvider(metadataStore, identity.keyring, feedStore, partyKey);
+    const writableFeed = await partyFeedProvider.createOrOpenWritableFeed();
 
-    return new DataParty(
+    const party = new DataParty(
       partyKey,
       modelFactory,
       snapshotStore,
       partyFeedProvider,
+      metadataStore,
       identity.createCredentialsSigner(),
       identity.preferences,
-      networkManager,
-      hints
+      networkManager
     );
+    party._setFeedHints([...feedHints, writableFeed.key]);
+    return party;
   };
 
   test('open & close', async () => {
@@ -62,7 +65,7 @@ describe('DataParty', () => {
     await party.open();
 
     const feed = await party.getWriteFeed();
-    await party.writeCredentialsMessage(createPartyGenesisMessage(
+    await party.credentialsWriter.write(createPartyGenesisMessage(
       keyring,
       partyKey,
       feed.key,
@@ -74,6 +77,33 @@ describe('DataParty', () => {
     await party.close();
   });
 
+  test('data is immediately available after re-opening', async () => {
+    const keyring = new Keyring();
+    const identity = await createTestIdentityCredentials(keyring);
+    const partyKey = await keyring.createKeyRecord({ type: KeyType.PARTY });
+    const party = await createParty(identity, partyKey.publicKey, []);
+    await party.open();
+
+    const feed = await party.getWriteFeed();
+    await party.credentialsWriter.write(createPartyGenesisMessage(
+      keyring,
+      partyKey,
+      feed.key,
+      partyKey
+    ));
+
+    for (let i = 0; i < 10; i++) {
+      await party.database.createItem({ type: 'test:item' });
+    }
+
+    await party.close();
+    await party.open();
+
+    expect(party.database.select({ type: 'test:item' }).exec().entities).toHaveLength(10);
+
+    await party.close();
+  });
+
   test('authenticates its own credentials', async () => {
     const keyring = new Keyring();
     const identity = await createTestIdentityCredentials(keyring);
@@ -82,14 +112,15 @@ describe('DataParty', () => {
     const party = await createParty(identity, partyKey.publicKey, []);
     await party.open();
     const feed = await party.getWriteFeed();
-    await party.writeCredentialsMessage(createPartyGenesisMessage(
+    await party.credentialsWriter.write(createPartyGenesisMessage(
       keyring,
       partyKey,
       feed.key,
       partyKey
     ));
+    await party.processor.feedAdded.waitForCount(1);
 
-    const authenticator = createAuthenticator(party.processor, identity.createCredentialsSigner());
+    const authenticator = createAuthenticator(party.processor, identity.createCredentialsSigner(), party.credentialsWriter);
     const credentialsProvider = createCredentialsProvider(identity.createCredentialsSigner(), party.key, feed.key);
 
     const wrappedCredentials = haloCodec.decode(credentialsProvider.get());
@@ -106,13 +137,15 @@ describe('DataParty', () => {
     const party = await createParty(identityA, partyKey.publicKey, []);
     await party.open();
     const feed = await party.getWriteFeed();
-    await party.writeCredentialsMessage(createPartyGenesisMessage(
+    await party.credentialsWriter.write(createPartyGenesisMessage(
       keyring,
       partyKey,
       feed.key,
       partyKey
     ));
-    const authenticator = createAuthenticator(party.processor, identityA.createCredentialsSigner());
+    await party.processor.feedAdded.waitForCount(1);
+
+    const authenticator = createAuthenticator(party.processor, identityA.createCredentialsSigner(), party.credentialsWriter);
 
     const identityB = await deriveTestDeviceCredentials(identityA);
     const credentialsProvider = createCredentialsProvider(identityB.createCredentialsSigner(), party.key, feed.key);
@@ -131,13 +164,13 @@ describe('DataParty', () => {
     const partyA = await createParty(identityA, partyKey.publicKey, []);
     await partyA.open();
     const feedA = await partyA.getWriteFeed();
-    await partyA.writeCredentialsMessage(createPartyGenesisMessage(
+    await partyA.credentialsWriter.write(createPartyGenesisMessage(
       keyring,
       partyKey,
       feedA.key,
       partyKey
     ));
-    await partyA.writeCredentialsMessage(createKeyAdmitMessage(
+    await partyA.credentialsWriter.write(createKeyAdmitMessage(
       keyring,
       partyKey.publicKey,
       identityA.identityKey,
@@ -145,9 +178,7 @@ describe('DataParty', () => {
     ));
 
     const identityB = await deriveTestDeviceCredentials(identityA);
-    const partyB = await createParty(identityB, partyKey.publicKey, [
-      { type: KeyType.FEED, publicKey: feedA.key }
-    ]);
+    const partyB = await createParty(identityB, partyKey.publicKey, [feedA.key]);
     await partyB.open();
 
     await partyA.database.createItem({ type: 'test:item-a' });
@@ -167,13 +198,13 @@ describe('DataParty', () => {
     const partyA = await createParty(identityA, partyKeyA.publicKey, []);
     await partyA.open();
     const feedA = await partyA.getWriteFeed();
-    await partyA.writeCredentialsMessage(createPartyGenesisMessage(
+    await partyA.credentialsWriter.write(createPartyGenesisMessage(
       identityA.keyring,
       partyKeyA,
       feedA.key,
       partyKeyA
     ));
-    await partyA.writeCredentialsMessage(createKeyAdmitMessage(
+    await partyA.credentialsWriter.write(createKeyAdmitMessage(
       identityA.keyring,
       partyKeyA.publicKey,
       identityA.identityKey,

package/src/parties/data-party.ts

@@ -5,19 +5,19 @@
 import assert from 'assert';
 
 import { synchronized, Event } from '@dxos/async';
-import { KeyHint, Message as HaloMessage } from '@dxos/credentials';
 import { PublicKey } from '@dxos/crypto';
 import { timed } from '@dxos/debug';
-import { PartyKey, PartySnapshot, Timeframe, WriteReceipt } from '@dxos/echo-protocol';
+import { PartyKey, PartySnapshot, Timeframe } from '@dxos/echo-protocol';
 import { FeedDescriptor } from '@dxos/feed-store';
 import { ModelFactory } from '@dxos/model-factory';
 import { NetworkManager } from '@dxos/network-manager';
 import { ObjectModel } from '@dxos/object-model';
 
-import { Database, Item, ResultSet } from '../api';
+import { ResultSet } from '../api';
 import { ActivationOptions, PartyPreferences, Preferences } from '../halo';
 import { InvitationFactory } from '../invitations';
-import { PartyFeedProvider, PartyProtocolFactory, PartyCore, PartyOptions } from '../pipeline';
+import { Database, Item } from '../packlets/database';
+import { PartyFeedProvider, PartyProtocolFactory, PartyPipeline, PipelineOptions, MetadataStore } from '../pipeline';
 import { createAuthPlugin, createOfflineInvitationPlugin, createAuthenticator, createCredentialsProvider } from '../protocol';
 import { CredentialsSigner } from '../protocol/credentials-signer';
 import { createReplicatorPlugin } from '../protocol/replicator-plugin';
@@ -42,31 +42,31 @@ export interface PartyMember {
 export class DataParty {
   public readonly update = new Event<void>();
 
-  private readonly _partyCore: PartyCore;
+  private readonly _partyCore: PartyPipeline;
   private readonly _preferences?: PartyPreferences;
   private _invitationManager?: InvitationFactory;
   private _protocol?: PartyProtocolFactory;
+  private _feedHints: PublicKey[] = []
 
   constructor (
     partyKey: PartyKey,
     modelFactory: ModelFactory,
     snapshotStore: SnapshotStore,
     private readonly _feedProvider: PartyFeedProvider,
+    private readonly _metadataStore: MetadataStore,
     private readonly _credentialsSigner: CredentialsSigner,
     // TODO(dmaretskyi): Pull this out to a higher level. Should preferences be part of client API instead?
     private readonly _profilePreferences: Preferences | undefined,
     private readonly _networkManager: NetworkManager,
-    private readonly _hints: KeyHint[] = [],
-    _initialTimeframe?: Timeframe,
-    _options: PartyOptions = {}
+    private readonly _initialTimeframe?: Timeframe,
+    _options: PipelineOptions = {}
   ) {
-    this._partyCore = new PartyCore(
+    this._partyCore = new PartyPipeline(
       partyKey,
       _feedProvider,
       modelFactory,
       snapshotStore,
       this._credentialsSigner.getIdentityKey().publicKey,
-      _initialTimeframe,
       _options
     );
 
@@ -127,6 +127,10 @@ export class DataParty {
     return this._invitationManager;
   }
 
+  get credentialsWriter () {
+    return this._partyCore.credentialsWriter;
+  }
+
   get title () {
     return this._preferences?.getLastKnownTitle();
   }
@@ -137,6 +141,13 @@ export class DataParty {
     await this._preferences?.setLastKnownTitle(title);
   }
 
+  /**
+   * @internal
+   */
+  _setFeedHints (feedHints: PublicKey[]) {
+    this._feedHints = feedHints;
+  }
+
   /**
    * Opens the pipeline and connects the streams.
    */
@@ -147,11 +158,25 @@ export class DataParty {
       return this;
     }
 
-    await this._partyCore.open(this._hints);
+    // TODO(dmaretskyi): May be undefined in some tests.
+    const party = this._metadataStore.getParty(this._partyCore.key);
+
+    await this._partyCore.open({
+      feedHints: this._feedHints,
+      initialTimeframe: this._initialTimeframe,
+      targetTimeframe: party?.latestTimeframe
+    });
+
+    // Keep updating latest reached timeframe in the metadata.
+    // This timeframe will be waited for when opening the party next time.
+    this._partyCore.timeframeUpdate.on(timeframe => {
+      void this._metadataStore.setTimeframe(this._partyCore.key, timeframe);
+    });
 
     this._invitationManager = new InvitationFactory(
       this._partyCore.processor,
       this._credentialsSigner,
+      this._partyCore.credentialsWriter,
       this._networkManager
     );
 
@@ -171,7 +196,7 @@ export class DataParty {
 
     await this._protocol.start([
       createReplicatorPlugin(this._feedProvider),
-      createAuthPlugin(createAuthenticator(this._partyCore.processor, this._credentialsSigner), deviceKey.publicKey),
+      createAuthPlugin(createAuthenticator(this._partyCore.processor, this._credentialsSigner, this.credentialsWriter), deviceKey.publicKey),
       createOfflineInvitationPlugin(this._invitationManager, deviceKey.publicKey)
     ]);
 
@@ -191,6 +216,9 @@ export class DataParty {
       return this;
     }
 
+    // Save the latest reached timeframe.
+    await this._metadataStore.setTimeframe(this._partyCore.key, this._partyCore.timeframe);
+
     await this._partyCore.close();
     await this._protocol?.stop();
 
@@ -210,10 +238,6 @@ export class DataParty {
     return this._feedProvider.getFeeds();
   }
 
-  writeCredentialsMessage (message: HaloMessage): Promise<WriteReceipt> {
-    return this._partyCore.writeCredentialsMessage(message);
-  }
-
   get isActive (): boolean {
     assert(this._preferences, 'PartyActivator required');
     return this._preferences.isActive;