@dxos/echo-db 2.33.5-dev.ea3876ba → 2.33.5-dev.fee8f5fe

package/src/invitations/invitation-factory.ts

@@ -4,11 +4,12 @@
 
 import assert from 'assert';
 
-import { createPartyInvitationMessage } from '@dxos/credentials';
+import { createPartyInvitationMessage, Message as HaloMessage } from '@dxos/credentials';
 import { PublicKey } from '@dxos/crypto';
+import { FeedWriter } from '@dxos/echo-protocol';
 import { NetworkManager } from '@dxos/network-manager';
 
-import { PartyProcessor } from '../pipeline';
+import { PartyStateProvider } from '../pipeline';
 import { CredentialsSigner } from '../protocol/credentials-signer';
 import { defaultInvitationAuthenticator, InvitationAuthenticator, InvitationOptions } from './common';
 import { GreetingResponder } from './greeting-responder';
@@ -19,10 +20,11 @@ import { InvitationDescriptor, InvitationDescriptorType } from './invitation-des
  */
 export class InvitationFactory {
   constructor (
-    private readonly _partyProcessor: PartyProcessor,
+    private readonly _partyProcessor: PartyStateProvider,
     // This needs to be a provider in case this is a backend for the HALO party.
     // Then the identity would be changed after this is instantiated.
     private readonly _credentialsSigner: CredentialsSigner,
+    private readonly _credentialsWriter: FeedWriter<HaloMessage>,
     private readonly _networkManager: NetworkManager
   ) {}
 
@@ -42,7 +44,7 @@ export class InvitationFactory {
       this._credentialsSigner.getDeviceSigningKeys()
     );
 
-    await this._partyProcessor.writeHaloMessage(invitationMessage);
+    await this._credentialsWriter.write(invitationMessage);
 
     return new InvitationDescriptor(
       InvitationDescriptorType.OFFLINE,
@@ -60,7 +62,8 @@ export class InvitationFactory {
     const responder = new GreetingResponder(
       this._networkManager,
       this._partyProcessor,
-      this._credentialsSigner
+      this._credentialsSigner,
+      this._credentialsWriter
     );
 
     const { secretValidator, secretProvider } = authenticationDetails;

package/src/invitations/offline-invitation-claimer.ts

@@ -23,7 +23,7 @@ import {
 import { keyToBuffer, keyToString, PublicKey, randomBytes } from '@dxos/crypto';
 import { FullyConnectedTopology, NetworkManager } from '@dxos/network-manager';
 
-import { InvalidInvitationError } from '../errors';
+import { InvalidInvitationError } from '../packlets/errors';
 import { CredentialsSigner } from '../protocol';
 import { greetingProtocolProvider } from './greeting-protocol-provider';
 import { GreetingState } from './greeting-responder';
@@ -70,8 +70,8 @@ export class OfflineInvitationClaimer {
     this._greeterPlugin = new GreetingCommandPlugin(localPeerId, async () => false);
 
     log('Connecting');
-    const peerJoinedWaiter = waitForEvent(this._greeterPlugin, 'peer:joined',
-      () => this._greeterPlugin?.peers.length, timeout);
+    const peerJoinedWaiter = waitForEvent(
+      this._greeterPlugin, 'peer:joined', () => !!this._greeterPlugin?.peers.length, timeout);
 
     await this._networkManager.joinProtocolSwarm({
       topic: PublicKey.from(swarmKey),
@@ -170,19 +170,17 @@ export class OfflineInvitationClaimer {
 
   // The secretProvider should provide an `Auth` message signed directly by the Identity key.
   static createSecretProvider (credentials: CredentialsSigner): SecretProvider {
-    return async (info?: SecretInfo) => {
-      return Buffer.from(codec.encode(
-        /* The signed portion of the Auth message includes the ID and authNonce provided
+    return async (info?: SecretInfo) => Buffer.from(codec.encode(
+      /* The signed portion of the Auth message includes the ID and authNonce provided
          * by the `info` object. These values will be validated on the other end.
          */
-        createAuthMessage(
-          credentials.signer,
+      createAuthMessage(
+        credentials.signer,
           info!.id.value,
           credentials.getIdentityKey(),
           credentials.getDeviceSigningKeys(),
           undefined,
           info!.authNonce.value)
-      ));
-    };
+    ));
   }
 }

package/src/{database → packlets/database}/data-mirror.test.ts

@@ -11,10 +11,10 @@ import { EchoEnvelope, MockFeedWriter, Timeframe } from '@dxos/echo-protocol';
 import { ModelFactory } from '@dxos/model-factory';
 import { ObjectModel } from '@dxos/object-model';
 
-import { Item } from '../api';
 import { DataMirror } from './data-mirror';
 import { DataServiceHost } from './data-service-host';
 import { DataServiceRouter } from './data-service-router';
+import { Item } from './item';
 import { ItemDemuxer } from './item-demuxer';
 import { ItemManager } from './item-manager';
 
@@ -26,8 +26,8 @@ describe('DataMirror', () => {
     const itemManager = new ItemManager(modelFactory, PublicKey.random(), feed);
     const itemDemuxer = new ItemDemuxer(itemManager, modelFactory, { snapshots: true });
 
-    const stream = itemDemuxer.open();
-    feed.written.on(([msg, meta]) => stream.write({
+    const process = itemDemuxer.open();
+    feed.written.on(([msg, meta]) => process({
       data: msg,
       meta: { ...meta, memberKey: PublicKey.random(), timeframe: new Timeframe() }
     } as any));

package/src/{database → packlets/database}/data-mirror.ts

@@ -10,7 +10,7 @@ import { failUndefined } from '@dxos/debug';
 import { DataService } from '@dxos/echo-protocol';
 import { Model } from '@dxos/model-factory';
 
-import { Entity } from '../api';
+import { Entity } from './entity';
 import { ItemManager } from './item-manager';
 
 const log = debug('dxos:echo-db:data-mirror');

package/src/{database → packlets/database}/data-service-host.ts

@@ -18,10 +18,11 @@ import {
   SubscribeEntityStreamResponse
 } from '@dxos/echo-protocol';
 
-import { Item, Link } from '../api';
 import { EntityNotFoundError } from '../errors';
+import { Item } from './item';
 import { ItemDemuxer } from './item-demuxer';
 import { ItemManager } from './item-manager';
+import { Link } from './link';
 
 const log = debug('dxos:echo-db:data-service-host');
 

package/src/{database → packlets/database}/database-backend.ts

@@ -10,7 +10,7 @@ import { ModelFactory } from '@dxos/model-factory';
 
 import { DataMirror } from './data-mirror';
 import { DataServiceHost } from './data-service-host';
-import { ItemDemuxer, ItemDemuxerOptions } from './item-demuxer';
+import { EchoProcessor, ItemDemuxer, ItemDemuxerOptions } from './item-demuxer';
 import { ItemManager } from './item-manager';
 
 const log = debug('dxos:echo-db:database-backend');
@@ -42,12 +42,11 @@ export interface DatabaseBackend {
  * Write operations result in mutations being written to the outgoing stream.
  */
 export class FeedDatabaseBackend implements DatabaseBackend {
-  private _itemDemuxerInboundStream!: NodeJS.WritableStream
+  private _echoProcessor!: EchoProcessor;
   private _itemManager!: ItemManager;
   private _itemDemuxer!: ItemDemuxer;
 
   constructor (
-    private readonly _inboundStream: NodeJS.ReadableStream,
     private readonly _outboundStream: FeedWriter<EchoEnvelope> | undefined,
     private readonly _snapshot?: DatabaseSnapshot,
     private readonly _options: ItemDemuxerOptions = {}
@@ -56,16 +55,18 @@ export class FeedDatabaseBackend implements DatabaseBackend {
   async open (itemManager: ItemManager, modelFactory: ModelFactory) {
     this._itemManager = itemManager;
     this._itemDemuxer = new ItemDemuxer(itemManager, modelFactory, this._options);
-    this._itemDemuxerInboundStream = this._itemDemuxer.open();
-    this._inboundStream.pipe(this._itemDemuxerInboundStream);
+    this._echoProcessor = this._itemDemuxer.open();
 
     if (this._snapshot) {
       await this._itemDemuxer.restoreFromSnapshot(this._snapshot);
     }
   }
 
+  get echoProcessor () {
+    return this._echoProcessor;
+  }
+
   async close () {
-    this._inboundStream?.unpipe(this._itemDemuxerInboundStream);
   }
 
   get isReadOnly (): boolean {

package/src/{api → packlets/database}/database.test.ts

@@ -10,9 +10,10 @@ import { ModelFactory, TestListModel } from '@dxos/model-factory';
 import { ObjectModel } from '@dxos/object-model';
 import { afterTest } from '@dxos/testutils';
 
-import { createInMemoryDatabase, createRemoteDatabaseFromDataServiceHost, DataServiceHost } from '../database';
+import { DataServiceHost } from './data-service-host';
 import { Item } from './item';
 import { ItemFilterDeleted } from './selection';
+import { createInMemoryDatabase, createRemoteDatabaseFromDataServiceHost } from './testing';
 
 const OBJECT_ORG = 'example:object/org';
 const OBJECT_PERSON = 'example:object/person';

package/src/{api → packlets/database}/database.ts

@@ -10,9 +10,11 @@ import { ItemID, ItemType } from '@dxos/echo-protocol';
 import { Model, ModelConstructor, ModelFactory, validateModelClass } from '@dxos/model-factory';
 import { ObjectModel } from '@dxos/object-model';
 
-import { DatabaseBackend, DataServiceHost, ItemManager } from '../database';
+import { DataServiceHost } from './data-service-host';
+import { DatabaseBackend } from './database-backend';
 import { Entity } from './entity';
 import { Item } from './item';
+import { ItemManager } from './item-manager';
 import { Link } from './link';
 import { RootFilter, Selection, createSelection } from './selection';
 

package/src/{api → packlets/database}/entity.ts

@@ -7,7 +7,7 @@ import { ItemID, ItemType } from '@dxos/echo-protocol';
 import { Model, ModelMeta, StateManager } from '@dxos/model-factory';
 import { SubscriptionGroup } from '@dxos/util';
 
-import { ItemManager } from '../database';
+import { ItemManager } from './item-manager';
 
 /**
  * Base class for all ECHO entitities.

package/src/{database → packlets/database}/index.ts

@@ -10,3 +10,8 @@ export * from './item-demuxer';
 export * from './item-manager';
 export * from './testing';
 export * from './timeframe-clock';
+export * from './item';
+export * from './link';
+export * from './entity';
+export * from './selection';
+export * from './database';

package/src/{database → packlets/database}/item-demuxer.test.ts

@@ -9,12 +9,11 @@ import { it as test } from 'mocha';
 import { latch } from '@dxos/async';
 import { createId, PublicKey } from '@dxos/crypto';
 import { checkType } from '@dxos/debug';
-import { createMockFeedWriterFromStream, EchoEnvelope, IEchoStream, MockFeedWriter, Timeframe } from '@dxos/echo-protocol';
-import { createTransform } from '@dxos/feed-store';
+import { EchoEnvelope, MockFeedWriter, Timeframe } from '@dxos/echo-protocol';
 import { ModelFactory, TestModel } from '@dxos/model-factory';
 import { ObjectModel } from '@dxos/object-model';
 
-import { Item } from '../api';
+import { Item } from './item';
 import { ItemDemuxer } from './item-demuxer';
 import { ItemManager } from './item-manager';
 
@@ -32,7 +31,7 @@ describe('Item demuxer', () => {
     const itemDemuxer = new ItemDemuxer(itemManager, modelFactory);
 
     const inboundStream = itemDemuxer.open();
-    feedWriter.written.on(([msg, meta]) => inboundStream.write({
+    feedWriter.written.on(([msg, meta]) => inboundStream({
       data: msg,
       meta: { ...meta, memberKey }
     } as any));
@@ -96,28 +95,31 @@ describe('Item demuxer', () => {
     const modelFactory = new ModelFactory()
       .registerModel(ObjectModel);
 
-    const writeStream = createTransform<EchoEnvelope, IEchoStream>(
-      async (message: EchoEnvelope): Promise<IEchoStream> => ({
-        meta: {
-          feedKey: PublicKey.random(),
-          memberKey: PublicKey.random(),
-          seq: 0,
-          timeframe: new Timeframe()
-        },
-        data: message
-      })
-    );
-    const itemManager = new ItemManager(modelFactory, PublicKey.random(), createMockFeedWriterFromStream(writeStream));
+    const itemManager = new ItemManager(modelFactory, PublicKey.random(), {
+      write: async (message) => {
+        void processEchoMessage(message);
+        return { feedKey: PublicKey.random(), seq: 0 };
+      }
+    });
     const itemDemuxer = new ItemDemuxer(itemManager, modelFactory);
-    writeStream.pipe(itemDemuxer.open());
+    const processor = itemDemuxer.open();
+    const processEchoMessage = (message: EchoEnvelope) => processor({
+      meta: {
+        feedKey: PublicKey.random(),
+        memberKey: PublicKey.random(),
+        seq: 0,
+        timeframe: new Timeframe()
+      },
+      data: message
+    });
 
-    writeStream.write(checkType<EchoEnvelope>({
+    void processEchoMessage(checkType<EchoEnvelope>({
       itemId: 'foo',
       genesis: {
         modelType: TestModel.meta.type
       }
     }));
-    writeStream.write(checkType<EchoEnvelope>({
+    void processEchoMessage(checkType<EchoEnvelope>({
       itemId: 'bar',
       genesis: {
         modelType: ObjectModel.meta.type

package/src/{database → packlets/database}/item-demuxer.ts

@@ -8,12 +8,12 @@ import debug from 'debug';
 import { Event } from '@dxos/async';
 import { failUndefined } from '@dxos/debug';
 import { DatabaseSnapshot, IEchoStream, ItemID, ItemSnapshot, LinkSnapshot } from '@dxos/echo-protocol';
-import { createWritable } from '@dxos/feed-store';
 import { Model, ModelFactory, ModelMessage } from '@dxos/model-factory';
-import { jsonReplacer } from '@dxos/util';
 
-import { Entity, Item, Link } from '../api';
+import { Entity } from './entity';
+import { Item } from './item';
 import { ItemManager, ModelConstructionOptions } from './item-manager';
+import { Link } from './link';
 
 const log = debug('dxos:echo-db:item-demuxer');
 
@@ -21,6 +21,8 @@ export interface ItemDemuxerOptions {
   snapshots?: boolean
 }
 
+export type EchoProcessor = (message: IEchoStream) => Promise<void>
+
 /**
  * Creates a stream that consumes `IEchoStream` messages and routes them to the associated items.
  * @param itemManager
@@ -34,7 +36,7 @@ export class ItemDemuxer {
     private readonly _options: ItemDemuxerOptions = {}
   ) {}
 
-  open (): NodeJS.WritableStream {
+  open (): EchoProcessor {
     this._modelFactory.registered.on(async model => {
       for (const item of this._itemManager.getUninitializedEntities()) {
         if (item._stateManager.modelType === model.meta.type) {
@@ -45,8 +47,7 @@ export class ItemDemuxer {
 
     // TODO(burdon): Factor out.
     // TODO(burdon): Should this implement some "back-pressure" (hints) to the PartyProcessor?
-    return createWritable<IEchoStream>(async (message: IEchoStream) => {
-      log('Reading:', JSON.stringify(message, jsonReplacer));
+    return async (message: IEchoStream) => {
       const { data: { itemId, genesis, itemMutation, mutation, snapshot }, meta } = message;
       assert(itemId);
 
@@ -111,7 +112,7 @@ export class ItemDemuxer {
       }
 
       this.mutation.emit(message);
-    });
+    };
   }
 
   createSnapshot (): DatabaseSnapshot {
@@ -187,7 +188,7 @@ export class ItemDemuxer {
  * Sort based on parents.
  * @param items
  */
-export function sortItemsTopologically (items: ItemSnapshot[]): ItemSnapshot[] {
+export const sortItemsTopologically = (items: ItemSnapshot[]): ItemSnapshot[] => {
   const snapshots: ItemSnapshot[] = [];
   const seenIds = new Set<ItemID>();
 
@@ -206,4 +207,4 @@ export function sortItemsTopologically (items: ItemSnapshot[]): ItemSnapshot[] {
   }
 
   return snapshots;
-}
+};

package/src/{database → packlets/database}/item-manager.ts

@@ -11,8 +11,10 @@ import { timed } from '@dxos/debug';
 import { EchoEnvelope, FeedWriter, ItemID, ItemType, mapFeedWriter, ModelSnapshot } from '@dxos/echo-protocol';
 import { Model, ModelFactory, ModelMessage, ModelType, StateManager } from '@dxos/model-factory';
 
-import { Entity, Item, Link } from '../api';
 import { UnknownModelError } from '../errors';
+import { Entity } from './entity';
+import { Item } from './item';
+import { Link } from './link';
 
 const log = debug('dxos:echo-db:item-manager');
 
@@ -375,7 +377,7 @@ export class ItemManager {
 /**
  * Returns a new event that groups all of the updates emitted during single tick into a single event emission.
  */
-function debounceEntityUpdateEvent (event: Event<Entity<any>>): Event<Entity<any>[]> {
+const debounceEntityUpdateEvent = (event: Event<Entity<any>>): Event<Entity<any>[]> => {
   const debouncedEvent = new Event<Entity<any>[]>();
 
   let firing = false;
@@ -395,4 +397,4 @@ function debounceEntityUpdateEvent (event: Event<Entity<any>>): Event<Entity<any
   }));
 
   return debouncedEvent;
-}
+};

package/src/{api → packlets/database}/item.ts

@@ -7,10 +7,10 @@ import debug from 'debug';
 import { EchoEnvelope, ItemID, ItemMutation, ItemType, FeedWriter } from '@dxos/echo-protocol';
 import { Model, StateManager } from '@dxos/model-factory';
 
-import { ItemManager } from '../database';
 import { Entity } from './entity';
+import { ItemManager } from './item-manager';
 import type { Link } from './link';
-import { Selection, createItemSelection } from './selection';
+import { createItemSelection, Selection } from './selection';
 
 const log = debug('dxos:echo-db:item');
 

package/src/{api → packlets/database}/link.ts

@@ -7,9 +7,9 @@ import assert from 'assert';
 import { ItemID, ItemType } from '@dxos/echo-protocol';
 import { Model, StateManager } from '@dxos/model-factory';
 
-import { ItemManager } from '../database';
 import { Entity } from './entity';
 import { Item } from './item';
+import { ItemManager } from './item-manager';
 
 export interface LinkData {
   sourceId: ItemID

package/src/{api → packlets/database}/selection/queries.ts

@@ -84,25 +84,21 @@ export const itemFilterToPredicate = (filter: ItemFilter | ItemIdFilter): Predic
   }
 };
 
-export const linkFilterToPredicate = (filter: LinkFilter): Predicate<Link> => {
-  return link => (!filter.type || testOneOrMultiple(filter.type, link.type));
-};
+export const linkFilterToPredicate = (filter: LinkFilter): Predicate<Link> => link => (!filter.type || testOneOrMultiple(filter.type, link.type));
 
 export const createQueryOptionsFilter = ({
   deleted = ItemFilterDeleted.HIDE_DELETED
-}: QueryOptions): Predicate<Entity> => {
-  return entity => {
-    if (entity.model === null) {
-      return false;
-    }
-
-    switch (deleted) {
-      case ItemFilterDeleted.HIDE_DELETED:
-        return !(entity instanceof Item) || !entity.deleted;
-      case ItemFilterDeleted.SHOW_DELETED:
-        return true;
-      case ItemFilterDeleted.SHOW_DELETED_ONLY:
-        return entity instanceof Item && entity.deleted;
-    }
-  };
+}: QueryOptions): Predicate<Entity> => entity => {
+  if (entity.model === null) {
+    return false;
+  }
+
+  switch (deleted) {
+    case ItemFilterDeleted.HIDE_DELETED:
+      return !(entity instanceof Item) || !entity.deleted;
+    case ItemFilterDeleted.SHOW_DELETED:
+      return true;
+    case ItemFilterDeleted.SHOW_DELETED_ONLY:
+      return entity instanceof Item && entity.deleted;
+  }
 };

package/src/{api → packlets/database}/selection/selection.test.ts

@@ -24,9 +24,7 @@ const modelFactory = new ModelFactory().registerModel(ObjectModel);
 
 const createModel = (id: ItemID) => modelFactory.createModel(ObjectModel.meta.type, id, {}, PublicKey.random());
 
-const createItem = (id: ItemID, type: ItemType, parent?: Item<any>) => {
-  return new Item(null as any, id, type, createModel(id), undefined, parent);
-};
+const createItem = (id: ItemID, type: ItemType, parent?: Item<any>) => new Item(null as any, id, type, createModel(id), undefined, parent);
 
 const createLink = (id: ItemID, type: ItemType, source: Item<any>, target: Item<any>) => {
   const link = new Link(null as any, id, type, createModel(id), {
@@ -262,17 +260,11 @@ describe('Selection', () => {
     test('complex reducer', () => {
       const query = createReducer({ numItems: 0, numLinks: 0 })
         .filter({ type: ITEM_ORG })
-        .call((items: Item[], { numItems, ...rest }) => {
-          return { ...rest, numItems: numItems + items.length, stage: 'a' };
-        })
+        .call((items: Item[], { numItems, ...rest }) => ({ ...rest, numItems: numItems + items.length, stage: 'a' }))
         .children({ type: ITEM_PROJECT })
-        .call((items: Item[], { numItems, ...rest }) => {
-          return { ...rest, numItems: numItems + items.length, stage: 'b' };
-        })
+        .call((items: Item[], { numItems, ...rest }) => ({ ...rest, numItems: numItems + items.length, stage: 'b' }))
         .links({ type: LINK_MEMBER })
-        .call((links: Link[], { numLinks, ...rest }) => {
-          return { ...rest, numLinks: numLinks + links.length, stage: 'c' };
-        })
+        .call((links: Link[], { numLinks, ...rest }) => ({ ...rest, numLinks: numLinks + links.length, stage: 'c' }))
         .target()
         .exec();
 

package/src/{api → packlets/database}/selection/selection.ts

@@ -57,13 +57,7 @@ export const createSelection = <R>(
  * @param update
  * @param value Initial reducer value.
  */
-export const createItemSelection = <R>(
-  root: Item<any>,
-  update: Event<Entity[]>,
-  value: R
-): Selection<Item<any>, R> => {
-  return new Selection(() => [[root], value], update, root, value !== undefined);
-};
+export const createItemSelection = <R>(root: Item<any>, update: Event<Entity[]>, value: R): Selection<Item<any>, R> => new Selection(() => [[root], value], update, root, value !== undefined);
 
 /**
  * Selections are used to construct database subscriptions.

package/src/{database → packlets/database}/testing.ts

@@ -2,25 +2,22 @@
 // Copyright 2021 DXOS.org
 //
 
-import { Readable } from 'stream';
-
 import { PublicKey } from '@dxos/crypto';
 import { EchoEnvelope, MockFeedWriter, Timeframe } from '@dxos/echo-protocol';
 import { ModelFactory } from '@dxos/model-factory';
 
-import { Database } from '../api';
 import { DataServiceHost } from './data-service-host';
 import { DataServiceRouter } from './data-service-router';
+import { Database } from './database';
 import { FeedDatabaseBackend, RemoteDatabaseBackend } from './database-backend';
 
 export const createInMemoryDatabase = async (modelFactory: ModelFactory) => {
   const feed = new MockFeedWriter<EchoEnvelope>();
-  const inboundStream = new Readable({ read () {}, objectMode: true });
-  feed.written.on(([data, meta]) => inboundStream.push({ data, meta: { ...meta, memberKey: PublicKey.random(), timeframe: new Timeframe([[meta.feedKey, meta.seq]]) } }));
-
+  const backend = new FeedDatabaseBackend(feed, undefined, { snapshots: true });
+  feed.written.on(([data, meta]) => backend.echoProcessor({ data, meta: { ...meta, memberKey: PublicKey.random(), timeframe: new Timeframe([[meta.feedKey, meta.seq]]) } }));
   const database = new Database(
     modelFactory,
-    new FeedDatabaseBackend(inboundStream, feed, undefined, { snapshots: true }),
+    backend,
     PublicKey.random()
   );
 

package/src/{database → packlets/database}/timeframe-clock.ts

@@ -28,4 +28,8 @@ export class TimeframeClock {
     const gaps = Timeframe.dependencies(timeframe, this._timeframe);
     return !gaps.isEmpty();
   }
+
+  async waitUntilReached (target: Timeframe) {
+    await this.update.waitForCondition(() => Timeframe.dependencies(target, this._timeframe).isEmpty());
+  }
 }