@dxos/echo-db 2.33.5-dev.4dcc5349 → 2.33.5-dev.8e90a74f

package/src/{database → packlets/database}/data-mirror.ts

@@ -10,7 +10,7 @@ import { failUndefined } from '@dxos/debug';
 import { DataService } from '@dxos/echo-protocol';
 import { Model } from '@dxos/model-factory';
 
-import { Entity } from '../api';
+import { Entity } from './entity';
 import { ItemManager } from './item-manager';
 
 const log = debug('dxos:echo-db:data-mirror');

package/src/{database → packlets/database}/data-service-host.ts

@@ -18,10 +18,11 @@ import {
   SubscribeEntityStreamResponse
 } from '@dxos/echo-protocol';
 
-import { Item, Link } from '../api';
 import { EntityNotFoundError } from '../errors';
+import { Item } from './item';
 import { ItemDemuxer } from './item-demuxer';
 import { ItemManager } from './item-manager';
+import { Link } from './link';
 
 const log = debug('dxos:echo-db:data-service-host');
 

package/src/{api → packlets/database}/database.test.ts

@@ -10,9 +10,10 @@ import { ModelFactory, TestListModel } from '@dxos/model-factory';
 import { ObjectModel } from '@dxos/object-model';
 import { afterTest } from '@dxos/testutils';
 
-import { createInMemoryDatabase, createRemoteDatabaseFromDataServiceHost, DataServiceHost } from '../database';
+import { DataServiceHost } from './data-service-host';
 import { Item } from './item';
 import { ItemFilterDeleted } from './selection';
+import { createInMemoryDatabase, createRemoteDatabaseFromDataServiceHost } from './testing';
 
 const OBJECT_ORG = 'example:object/org';
 const OBJECT_PERSON = 'example:object/person';

package/src/{api → packlets/database}/database.ts

@@ -10,9 +10,11 @@ import { ItemID, ItemType } from '@dxos/echo-protocol';
 import { Model, ModelConstructor, ModelFactory, validateModelClass } from '@dxos/model-factory';
 import { ObjectModel } from '@dxos/object-model';
 
-import { DatabaseBackend, DataServiceHost, ItemManager } from '../database';
+import { DataServiceHost } from './data-service-host';
+import { DatabaseBackend } from './database-backend';
 import { Entity } from './entity';
 import { Item } from './item';
+import { ItemManager } from './item-manager';
 import { Link } from './link';
 import { RootFilter, Selection, createSelection } from './selection';
 

package/src/{api → packlets/database}/entity.ts

@@ -7,7 +7,7 @@ import { ItemID, ItemType } from '@dxos/echo-protocol';
 import { Model, ModelMeta, StateManager } from '@dxos/model-factory';
 import { SubscriptionGroup } from '@dxos/util';
 
-import { ItemManager } from '../database';
+import { ItemManager } from './item-manager';
 
 /**
  * Base class for all ECHO entitities.

package/src/{database → packlets/database}/index.ts

@@ -10,3 +10,8 @@ export * from './item-demuxer';
 export * from './item-manager';
 export * from './testing';
 export * from './timeframe-clock';
+export * from './item';
+export * from './link';
+export * from './entity';
+export * from './selection';
+export * from './database';

package/src/{database → packlets/database}/item-demuxer.test.ts

@@ -14,7 +14,7 @@ import { createTransform } from '@dxos/feed-store';
 import { ModelFactory, TestModel } from '@dxos/model-factory';
 import { ObjectModel } from '@dxos/object-model';
 
-import { Item } from '../api';
+import { Item } from './item';
 import { ItemDemuxer } from './item-demuxer';
 import { ItemManager } from './item-manager';
 

package/src/{database → packlets/database}/item-demuxer.ts

@@ -10,10 +10,11 @@ import { failUndefined } from '@dxos/debug';
 import { DatabaseSnapshot, IEchoStream, ItemID, ItemSnapshot, LinkSnapshot } from '@dxos/echo-protocol';
 import { createWritable } from '@dxos/feed-store';
 import { Model, ModelFactory, ModelMessage } from '@dxos/model-factory';
-import { jsonReplacer } from '@dxos/util';
 
-import { Entity, Item, Link } from '../api';
+import { Entity } from './entity';
+import { Item } from './item';
 import { ItemManager, ModelConstructionOptions } from './item-manager';
+import { Link } from './link';
 
 const log = debug('dxos:echo-db:item-demuxer');
 
@@ -46,7 +47,6 @@ export class ItemDemuxer {
     // TODO(burdon): Factor out.
     // TODO(burdon): Should this implement some "back-pressure" (hints) to the PartyProcessor?
     return createWritable<IEchoStream>(async (message: IEchoStream) => {
-      log('Reading:', JSON.stringify(message, jsonReplacer));
       const { data: { itemId, genesis, itemMutation, mutation, snapshot }, meta } = message;
       assert(itemId);
 
@@ -187,7 +187,7 @@ export class ItemDemuxer {
  * Sort based on parents.
  * @param items
  */
-export function sortItemsTopologically (items: ItemSnapshot[]): ItemSnapshot[] {
+export const sortItemsTopologically = (items: ItemSnapshot[]): ItemSnapshot[] => {
   const snapshots: ItemSnapshot[] = [];
   const seenIds = new Set<ItemID>();
 
@@ -206,4 +206,4 @@ export function sortItemsTopologically (items: ItemSnapshot[]): ItemSnapshot[] {
   }
 
   return snapshots;
-}
+};

package/src/{database → packlets/database}/item-manager.ts

@@ -11,8 +11,10 @@ import { timed } from '@dxos/debug';
 import { EchoEnvelope, FeedWriter, ItemID, ItemType, mapFeedWriter, ModelSnapshot } from '@dxos/echo-protocol';
 import { Model, ModelFactory, ModelMessage, ModelType, StateManager } from '@dxos/model-factory';
 
-import { Entity, Item, Link } from '../api';
 import { UnknownModelError } from '../errors';
+import { Entity } from './entity';
+import { Item } from './item';
+import { Link } from './link';
 
 const log = debug('dxos:echo-db:item-manager');
 
@@ -375,7 +377,7 @@ export class ItemManager {
 /**
  * Returns a new event that groups all of the updates emitted during single tick into a single event emission.
  */
-function debounceEntityUpdateEvent (event: Event<Entity<any>>): Event<Entity<any>[]> {
+const debounceEntityUpdateEvent = (event: Event<Entity<any>>): Event<Entity<any>[]> => {
   const debouncedEvent = new Event<Entity<any>[]>();
 
   let firing = false;
@@ -395,4 +397,4 @@ function debounceEntityUpdateEvent (event: Event<Entity<any>>): Event<Entity<any
   }));
 
   return debouncedEvent;
-}
+};

package/src/{api → packlets/database}/item.ts

@@ -7,10 +7,10 @@ import debug from 'debug';
 import { EchoEnvelope, ItemID, ItemMutation, ItemType, FeedWriter } from '@dxos/echo-protocol';
 import { Model, StateManager } from '@dxos/model-factory';
 
-import { ItemManager } from '../database';
 import { Entity } from './entity';
+import { ItemManager } from './item-manager';
 import type { Link } from './link';
-import { Selection, createItemSelection } from './selection';
+import { createItemSelection, Selection } from './selection';
 
 const log = debug('dxos:echo-db:item');
 

package/src/{api → packlets/database}/link.ts

@@ -7,9 +7,9 @@ import assert from 'assert';
 import { ItemID, ItemType } from '@dxos/echo-protocol';
 import { Model, StateManager } from '@dxos/model-factory';
 
-import { ItemManager } from '../database';
 import { Entity } from './entity';
 import { Item } from './item';
+import { ItemManager } from './item-manager';
 
 export interface LinkData {
   sourceId: ItemID

package/src/{api → packlets/database}/selection/queries.ts

@@ -84,25 +84,21 @@ export const itemFilterToPredicate = (filter: ItemFilter | ItemIdFilter): Predic
   }
 };
 
-export const linkFilterToPredicate = (filter: LinkFilter): Predicate<Link> => {
-  return link => (!filter.type || testOneOrMultiple(filter.type, link.type));
-};
+export const linkFilterToPredicate = (filter: LinkFilter): Predicate<Link> => link => (!filter.type || testOneOrMultiple(filter.type, link.type));
 
 export const createQueryOptionsFilter = ({
   deleted = ItemFilterDeleted.HIDE_DELETED
-}: QueryOptions): Predicate<Entity> => {
-  return entity => {
-    if (entity.model === null) {
-      return false;
-    }
-
-    switch (deleted) {
-      case ItemFilterDeleted.HIDE_DELETED:
-        return !(entity instanceof Item) || !entity.deleted;
-      case ItemFilterDeleted.SHOW_DELETED:
-        return true;
-      case ItemFilterDeleted.SHOW_DELETED_ONLY:
-        return entity instanceof Item && entity.deleted;
-    }
-  };
+}: QueryOptions): Predicate<Entity> => entity => {
+  if (entity.model === null) {
+    return false;
+  }
+
+  switch (deleted) {
+    case ItemFilterDeleted.HIDE_DELETED:
+      return !(entity instanceof Item) || !entity.deleted;
+    case ItemFilterDeleted.SHOW_DELETED:
+      return true;
+    case ItemFilterDeleted.SHOW_DELETED_ONLY:
+      return entity instanceof Item && entity.deleted;
+  }
 };

package/src/{api → packlets/database}/selection/selection.test.ts

@@ -24,9 +24,7 @@ const modelFactory = new ModelFactory().registerModel(ObjectModel);
 
 const createModel = (id: ItemID) => modelFactory.createModel(ObjectModel.meta.type, id, {}, PublicKey.random());
 
-const createItem = (id: ItemID, type: ItemType, parent?: Item<any>) => {
-  return new Item(null as any, id, type, createModel(id), undefined, parent);
-};
+const createItem = (id: ItemID, type: ItemType, parent?: Item<any>) => new Item(null as any, id, type, createModel(id), undefined, parent);
 
 const createLink = (id: ItemID, type: ItemType, source: Item<any>, target: Item<any>) => {
   const link = new Link(null as any, id, type, createModel(id), {
@@ -262,17 +260,11 @@ describe('Selection', () => {
     test('complex reducer', () => {
       const query = createReducer({ numItems: 0, numLinks: 0 })
         .filter({ type: ITEM_ORG })
-        .call((items: Item[], { numItems, ...rest }) => {
-          return { ...rest, numItems: numItems + items.length, stage: 'a' };
-        })
+        .call((items: Item[], { numItems, ...rest }) => ({ ...rest, numItems: numItems + items.length, stage: 'a' }))
         .children({ type: ITEM_PROJECT })
-        .call((items: Item[], { numItems, ...rest }) => {
-          return { ...rest, numItems: numItems + items.length, stage: 'b' };
-        })
+        .call((items: Item[], { numItems, ...rest }) => ({ ...rest, numItems: numItems + items.length, stage: 'b' }))
         .links({ type: LINK_MEMBER })
-        .call((links: Link[], { numLinks, ...rest }) => {
-          return { ...rest, numLinks: numLinks + links.length, stage: 'c' };
-        })
+        .call((links: Link[], { numLinks, ...rest }) => ({ ...rest, numLinks: numLinks + links.length, stage: 'c' }))
         .target()
         .exec();
 

package/src/{api → packlets/database}/selection/selection.ts

@@ -57,13 +57,7 @@ export const createSelection = <R>(
  * @param update
  * @param value Initial reducer value.
  */
-export const createItemSelection = <R>(
-  root: Item<any>,
-  update: Event<Entity[]>,
-  value: R
-): Selection<Item<any>, R> => {
-  return new Selection(() => [[root], value], update, root, value !== undefined);
-};
+export const createItemSelection = <R>(root: Item<any>, update: Event<Entity[]>, value: R): Selection<Item<any>, R> => new Selection(() => [[root], value], update, root, value !== undefined);
 
 /**
  * Selections are used to construct database subscriptions.

package/src/{database → packlets/database}/testing.ts

@@ -8,14 +8,14 @@ import { PublicKey } from '@dxos/crypto';
 import { EchoEnvelope, MockFeedWriter, Timeframe } from '@dxos/echo-protocol';
 import { ModelFactory } from '@dxos/model-factory';
 
-import { Database } from '../api';
 import { DataServiceHost } from './data-service-host';
 import { DataServiceRouter } from './data-service-router';
+import { Database } from './database';
 import { FeedDatabaseBackend, RemoteDatabaseBackend } from './database-backend';
 
 export const createInMemoryDatabase = async (modelFactory: ModelFactory) => {
   const feed = new MockFeedWriter<EchoEnvelope>();
-  const inboundStream = new Readable({ read () {}, objectMode: true });
+  const inboundStream = new Readable({ read: () => {}, objectMode: true });
   feed.written.on(([data, meta]) => inboundStream.push({ data, meta: { ...meta, memberKey: PublicKey.random(), timeframe: new Timeframe([[meta.feedKey, meta.seq]]) } }));
 
   const database = new Database(

package/src/parties/data-party.test.ts

@@ -5,7 +5,7 @@
 import expect from 'expect';
 import { it as test } from 'mocha';
 
-import { createKeyAdmitMessage, createPartyGenesisMessage, defaultSecretProvider, KeyHint, Keyring, KeyType, codec as haloCodec } from '@dxos/credentials';
+import { createKeyAdmitMessage, createPartyGenesisMessage, defaultSecretProvider, Keyring, KeyType, codec as haloCodec } from '@dxos/credentials';
 import { PublicKey } from '@dxos/crypto';
 import { codec } from '@dxos/echo-protocol';
 import { FeedStore } from '@dxos/feed-store';
@@ -22,10 +22,12 @@ import { SnapshotStore } from '../snapshots';
 import { DataParty } from './data-party';
 
 describe('DataParty', () => {
-  const createParty = async (identity: IdentityCredentials, partyKey: PublicKey, hints: KeyHint[]) => {
-    const metadataStore = new MetadataStore(createStorage('metadata', StorageType.RAM));
-    const feedStore = new FeedStore(createStorage('feed', StorageType.RAM), { valueEncoding: codec });
-    const snapshotStore = new SnapshotStore(createStorage('snapshots', StorageType.RAM));
+  const createParty = async (identity: IdentityCredentials, partyKey: PublicKey, feedHints: PublicKey[]) => {
+
+    const storage = createStorage('', StorageType.RAM);
+    const snapshotStore = new SnapshotStore(storage.directory('snapshots'));
+    const metadataStore = new MetadataStore(storage.directory('metadata'));
+    const feedStore = new FeedStore(storage.directory('feed'), { valueEncoding: codec });
     const modelFactory = new ModelFactory().registerModel(ObjectModel);
     const networkManager = new NetworkManager();
     const partyFeedProvider = new PartyFeedProvider(metadataStore, identity.keyring, feedStore, partyKey);
@@ -38,7 +40,7 @@ describe('DataParty', () => {
       identity.createCredentialsSigner(),
       identity.preferences,
       networkManager,
-      hints
+      feedHints
     );
   };
 
@@ -59,8 +61,8 @@ describe('DataParty', () => {
     const party = await createParty(identity, partyKey.publicKey, []);
     await party.open();
 
-    const feed = await party.feedProvider.createOrOpenWritableFeed();
-    await party.processor.writeHaloMessage(createPartyGenesisMessage(
+    const feed = await party.getWriteFeed();
+    await party.credentialsWriter.write(createPartyGenesisMessage(
       keyring,
       partyKey,
       feed.key,
@@ -79,15 +81,16 @@ describe('DataParty', () => {
 
     const party = await createParty(identity, partyKey.publicKey, []);
     await party.open();
-    const feed = await party.feedProvider.createOrOpenWritableFeed();
-    await party.processor.writeHaloMessage(createPartyGenesisMessage(
+    const feed = await party.getWriteFeed();
+    await party.credentialsWriter.write(createPartyGenesisMessage(
       keyring,
       partyKey,
       feed.key,
       partyKey
     ));
+    await party.processor.feedAdded.waitForCount(1);
 
-    const authenticator = createAuthenticator(party.processor, identity.createCredentialsSigner());
+    const authenticator = createAuthenticator(party.processor, identity.createCredentialsSigner(), party.credentialsWriter);
     const credentialsProvider = createCredentialsProvider(identity.createCredentialsSigner(), party.key, feed.key);
 
     const wrappedCredentials = haloCodec.decode(credentialsProvider.get());
@@ -103,14 +106,16 @@ describe('DataParty', () => {
 
     const party = await createParty(identityA, partyKey.publicKey, []);
     await party.open();
-    const feed = await party.feedProvider.createOrOpenWritableFeed();
-    await party.processor.writeHaloMessage(createPartyGenesisMessage(
+    const feed = await party.getWriteFeed();
+    await party.credentialsWriter.write(createPartyGenesisMessage(
       keyring,
       partyKey,
       feed.key,
       partyKey
     ));
-    const authenticator = createAuthenticator(party.processor, identityA.createCredentialsSigner());
+    await party.processor.feedAdded.waitForCount(1);
+
+    const authenticator = createAuthenticator(party.processor, identityA.createCredentialsSigner(), party.credentialsWriter);
 
     const identityB = await deriveTestDeviceCredentials(identityA);
     const credentialsProvider = createCredentialsProvider(identityB.createCredentialsSigner(), party.key, feed.key);
@@ -128,14 +133,14 @@ describe('DataParty', () => {
 
     const partyA = await createParty(identityA, partyKey.publicKey, []);
     await partyA.open();
-    const feedA = await partyA.feedProvider.createOrOpenWritableFeed();
-    await partyA.processor.writeHaloMessage(createPartyGenesisMessage(
+    const feedA = await partyA.getWriteFeed();
+    await partyA.credentialsWriter.write(createPartyGenesisMessage(
       keyring,
       partyKey,
       feedA.key,
       partyKey
     ));
-    await partyA.processor.writeHaloMessage(createKeyAdmitMessage(
+    await partyA.credentialsWriter.write(createKeyAdmitMessage(
       keyring,
       partyKey.publicKey,
       identityA.identityKey,
@@ -143,9 +148,7 @@ describe('DataParty', () => {
     ));
 
     const identityB = await deriveTestDeviceCredentials(identityA);
-    const partyB = await createParty(identityB, partyKey.publicKey, [
-      { type: KeyType.FEED, publicKey: feedA.key }
-    ]);
+    const partyB = await createParty(identityB, partyKey.publicKey, [feedA.key]);
     await partyB.open();
 
     await partyA.database.createItem({ type: 'test:item-a' });
@@ -164,14 +167,14 @@ describe('DataParty', () => {
 
     const partyA = await createParty(identityA, partyKeyA.publicKey, []);
     await partyA.open();
-    const feedA = await partyA.feedProvider.createOrOpenWritableFeed();
-    await partyA.processor.writeHaloMessage(createPartyGenesisMessage(
+    const feedA = await partyA.getWriteFeed();
+    await partyA.credentialsWriter.write(createPartyGenesisMessage(
       identityA.keyring,
       partyKeyA,
       feedA.key,
       partyKeyA
     ));
-    await partyA.processor.writeHaloMessage(createKeyAdmitMessage(
+    await partyA.credentialsWriter.write(createKeyAdmitMessage(
       identityA.keyring,
       partyKeyA.publicKey,
       identityA.identityKey,

package/src/parties/data-party.ts

@@ -5,20 +5,22 @@
 import assert from 'assert';
 
 import { synchronized, Event } from '@dxos/async';
-import { KeyHint } from '@dxos/credentials';
 import { PublicKey } from '@dxos/crypto';
 import { timed } from '@dxos/debug';
 import { PartyKey, PartySnapshot, Timeframe } from '@dxos/echo-protocol';
+import { FeedDescriptor } from '@dxos/feed-store';
 import { ModelFactory } from '@dxos/model-factory';
 import { NetworkManager } from '@dxos/network-manager';
 import { ObjectModel } from '@dxos/object-model';
 
-import { Database, Item, ResultSet } from '../api';
+import { ResultSet } from '../api';
 import { ActivationOptions, PartyPreferences, Preferences } from '../halo';
 import { InvitationFactory } from '../invitations';
+import { Database, Item } from '../packlets/database';
 import { PartyFeedProvider, PartyProtocolFactory, PartyCore, PartyOptions } from '../pipeline';
 import { createAuthPlugin, createOfflineInvitationPlugin, createAuthenticator, createCredentialsProvider } from '../protocol';
 import { CredentialsSigner } from '../protocol/credentials-signer';
+import { createReplicatorPlugin } from '../protocol/replicator-plugin';
 import { SnapshotStore } from '../snapshots';
 import { CONTACT_DEBOUNCE_INTERVAL } from './party-manager';
 
@@ -54,8 +56,8 @@ export class DataParty {
     // TODO(dmaretskyi): Pull this out to a higher level. Should preferences be part of client API instead?
     private readonly _profilePreferences: Preferences | undefined,
     private readonly _networkManager: NetworkManager,
-    private readonly _hints: KeyHint[] = [],
-    _initialTimeframe?: Timeframe,
+    private readonly _feedHints: PublicKey[] = [],
+    private readonly _initialTimeframe?: Timeframe,
     _options: PartyOptions = {}
   ) {
     this._partyCore = new PartyCore(
@@ -64,7 +66,6 @@ export class DataParty {
       modelFactory,
       snapshotStore,
       this._credentialsSigner.getIdentityKey().publicKey,
-      _initialTimeframe,
       _options
     );
 
@@ -125,14 +126,8 @@ export class DataParty {
     return this._invitationManager;
   }
 
-  // TODO(burdon): Remove?
-  get feedProvider (): PartyFeedProvider {
-    return this._feedProvider;
-  }
-
-  get preferences (): PartyPreferences {
-    assert(this._preferences, 'Preferences not available.');
-    return this._preferences;
+  get credentialsWriter () {
+    return this._partyCore.credentialsWriter;
   }
 
   get title () {
@@ -155,11 +150,15 @@ export class DataParty {
       return this;
     }
 
-    await this._partyCore.open(this._hints);
+    await this._partyCore.open({
+      feedHints: this._feedHints,
+      initialTimeframe: this._initialTimeframe
+    });
 
     this._invitationManager = new InvitationFactory(
       this._partyCore.processor,
       this._credentialsSigner,
+      this._partyCore.credentialsWriter,
       this._networkManager
     );
 
@@ -173,13 +172,13 @@ export class DataParty {
     this._protocol = new PartyProtocolFactory(
       this._partyCore.key,
       this._networkManager,
-      this._feedProvider,
       deviceKey.publicKey,
       createCredentialsProvider(this._credentialsSigner, this._partyCore.key, writeFeed.key)
     );
 
     await this._protocol.start([
-      createAuthPlugin(createAuthenticator(this._partyCore.processor, this._credentialsSigner), deviceKey.publicKey),
+      createReplicatorPlugin(this._feedProvider),
+      createAuthPlugin(createAuthenticator(this._partyCore.processor, this._credentialsSigner, this.credentialsWriter), deviceKey.publicKey),
       createOfflineInvitationPlugin(this._invitationManager, deviceKey.publicKey)
     ]);
 
@@ -210,6 +209,14 @@ export class DataParty {
     return this;
   }
 
+  async getWriteFeed (): Promise<FeedDescriptor> {
+    return this._feedProvider.createOrOpenWritableFeed();
+  }
+
+  getFeeds (): FeedDescriptor[] {
+    return this._feedProvider.getFeeds();
+  }
+
   get isActive (): boolean {
     assert(this._preferences, 'PartyActivator required');
     return this._preferences.isActive;