@dxos/echo-db 2.33.5-dev.b7c9d504 → 2.33.5-dev.bbe0c471

package/src/pipeline/party-processor.ts

@@ -12,22 +12,40 @@ import {
   PartyState,
   Message as HaloMessage,
   IdentityEventType,
-  PartyEventType
+  PartyEventType,
+  SignedMessage
 } from '@dxos/credentials';
 import { PublicKey } from '@dxos/crypto';
-import { FeedKey, FeedWriter, IHaloStream, PartyKey, HaloStateSnapshot, WriteReceipt } from '@dxos/echo-protocol';
+import { FeedKey, IHaloStream, PartyKey, HaloStateSnapshot } from '@dxos/echo-protocol';
 import { jsonReplacer } from '@dxos/util';
 
 const log = debug('dxos:echo-db:party-processor');
 
+export interface CredentialProcessor {
+  processMessage (message: IHaloStream): Promise<void>
+}
+
+export interface PartyStateProvider {
+  partyKey: PublicKey
+
+  /**
+   * Whether PartyGenesis was already processed.
+   */
+  genesisRequired: boolean
+  memberKeys: PublicKey[]
+  feedKeys: PublicKey[]
+  getFeedOwningMember (feedKey: FeedKey): PublicKey | undefined
+  isFeedAdmitted (feedKey: FeedKey): boolean
+
+  getOfflineInvitation (invitationID: Buffer): SignedMessage | undefined
+}
+
 /**
  * TODO(burdon): Wrapper/Bridge between HALO APIs.
  */
-export class PartyProcessor {
+export class PartyProcessor implements CredentialProcessor, PartyStateProvider {
   private readonly _state: PartyState;
 
-  private _outboundHaloStream: FeedWriter<HaloMessage> | undefined;
-
   readonly feedAdded = new Event<FeedKey>()
 
   public readonly keyOrInfoAdded = new Event<PublicKey>();
@@ -124,16 +142,6 @@ export class PartyProcessor {
     await this._state.processMessages([data]);
   }
 
-  setOutboundStream (stream: FeedWriter<HaloMessage>) {
-    this._outboundHaloStream = stream;
-  }
-
-  async writeHaloMessage (message: HaloMessage): Promise<WriteReceipt> {
-    assert(this._outboundHaloStream, 'Party is closed or read-only');
-    // TODO(marik-d): Wait for the message to be processed?
-    return this._outboundHaloStream.write(message);
-  }
-
   makeSnapshot (): HaloStateSnapshot {
     return {
       messages: this._haloMessages

package/src/pipeline/pipeline.test.ts

@@ -25,7 +25,7 @@ const log = debug('dxos:echo:pipeline:test');
 describe('pipeline', () => {
   test('streams', async () => {
     const storage = createStorage('', StorageType.RAM);
-    const feedStore = new FeedStore(storage, { valueEncoding: codec });
+    const feedStore = new FeedStore(storage.directory('feed'), { valueEncoding: codec });
     const feedKeys: Uint8Array[] = [];
     const feedSelector: FeedSelector = descriptor => !!feedKeys.find(key => descriptor.key.equals(key));
     const feedReadStream = new FeedStoreIterator(feedSelector, () => 0, new Timeframe());
@@ -87,7 +87,7 @@ describe('pipeline', () => {
 
   test('writing', async () => {
     const storage = createStorage('', StorageType.RAM);
-    const feedStore = new FeedStore(storage, { valueEncoding: codec });
+    const feedStore = new FeedStore(storage.directory('feed'), { valueEncoding: codec });
     const feedReadStream = new FeedStoreIterator(() => true, () => 0, new Timeframe());
 
     const { publicKey, secretKey } = createKeyPair();

package/src/pipeline/pipeline.ts

@@ -17,7 +17,7 @@ import { createReadable } from '@dxos/feed-store';
 import { jsonReplacer } from '@dxos/util';
 
 import { TimeframeClock } from '../database';
-import { PartyProcessor } from './party-processor';
+import { CredentialProcessor, PartyStateProvider } from './party-processor';
 
 interface Options {
   readLogger?: (msg: any) => void
@@ -55,17 +55,13 @@ export class Pipeline {
    * @param _options
    */
   constructor (
-    private readonly _partyProcessor: PartyProcessor,
+    private readonly _partyProcessor: CredentialProcessor & PartyStateProvider,
     private readonly _feedStorIterator: FeedStoreIterator,
     private readonly _timeframeClock: TimeframeClock,
     private readonly _feedWriter?: FeedWriter<FeedMessage>,
     private readonly _options: Options = {}
   ) {}
 
-  get partyKey () {
-    return this._partyProcessor.partyKey;
-  }
-
   get isOpen () {
     return this._inboundEchoStream !== undefined;
   }

package/src/protocol/auth-plugin.ts

@@ -9,6 +9,4 @@ import { Replicator } from '@dxos/protocol-plugin-replicator';
 /**
  * Creates authenticator network-protocol plugin that guards access to the replicator.
  */
-export function createAuthPlugin (authenticator: Authenticator, peerId: PublicKey) {
-  return new AuthPlugin(peerId.asBuffer(), authenticator, [Replicator.extension]);
-}
+export const createAuthPlugin = (authenticator: Authenticator, peerId: PublicKey) => new AuthPlugin(peerId.asBuffer(), authenticator, [Replicator.extension]);

package/src/protocol/authenticator.test.ts

@@ -6,7 +6,6 @@ import expect from 'expect';
 import { it as test } from 'mocha';
 
 import { createAuthMessage, createKeyAdmitMessage, createPartyGenesisMessage, Keyring, KeyType } from '@dxos/credentials';
-import { MockFeedWriter } from '@dxos/echo-protocol';
 
 import { PartyProcessor } from '../pipeline';
 import { createAuthenticator } from './authenticator';
@@ -23,8 +22,6 @@ describe('authenticator', () => {
     const signer = CredentialsSigner.createDirectDeviceSigner(keyring);
 
     const partyProcessor = new PartyProcessor(partyKey.publicKey);
-    const feed = new MockFeedWriter();
-    partyProcessor.setOutboundStream(feed);
     await partyProcessor.processMessage({
       data: createPartyGenesisMessage(
         keyring,
@@ -53,7 +50,7 @@ describe('authenticator', () => {
       meta: {} as any
     });
 
-    const authenticator = createAuthenticator(partyProcessor, signer);
+    const authenticator = createAuthenticator(partyProcessor, signer, null as any);
     const credential = createAuthMessage(
       keyring,
       partyKey.publicKey,

package/src/protocol/authenticator.ts

@@ -4,27 +4,29 @@
 
 import debug from 'debug';
 
-import { Authenticator, codec, createAuthMessage, createEnvelopeMessage, createFeedAdmitMessage, PartyAuthenticator } from '@dxos/credentials';
-import { FeedKey, PartyKey } from '@dxos/echo-protocol';
+import { Message as HaloMessage, Authenticator, codec, createAuthMessage, createEnvelopeMessage, createFeedAdmitMessage, PartyAuthenticator } from '@dxos/credentials';
+import { FeedKey, FeedWriter, PartyKey } from '@dxos/echo-protocol';
 
 import { PartyProcessor } from '../pipeline';
 import { CredentialsSigner } from './credentials-signer';
 
 const log = debug('dxos:echo-db:authenticator');
 
-export function createAuthenticator (partyProcessor: PartyProcessor, credentialsSigner: CredentialsSigner): Authenticator {
-  return new PartyAuthenticator(partyProcessor.state, async auth => {
-    if (auth.feedAdmit && auth.feedKey && !partyProcessor.isFeedAdmitted(auth.feedKey)) {
-      log(`Admitting feed of authenticated member: ${auth.feedKey}`);
-      await partyProcessor.writeHaloMessage(createEnvelopeMessage(
-        credentialsSigner.signer,
-        partyProcessor.partyKey,
-        auth.feedAdmit,
-        [credentialsSigner.getDeviceSigningKeys()]
-      ));
-    }
-  });
-}
+export const createAuthenticator = (
+  partyProcessor: PartyProcessor,
+  credentialsSigner: CredentialsSigner,
+  credentialsWriter: FeedWriter<HaloMessage>
+): Authenticator => new PartyAuthenticator(partyProcessor.state, async auth => {
+  if (auth.feedAdmit && auth.feedKey && !partyProcessor.isFeedAdmitted(auth.feedKey)) {
+    log(`Admitting feed of authenticated member: ${auth.feedKey}`);
+    await credentialsWriter.write(createEnvelopeMessage(
+      credentialsSigner.signer,
+      partyProcessor.partyKey,
+      auth.feedAdmit,
+      [credentialsSigner.getDeviceSigningKeys()]
+    ));
+  }
+});
 
 export interface CredentialsProvider {
   /**
@@ -33,24 +35,22 @@ export interface CredentialsProvider {
   get (): Buffer
 }
 
-export function createCredentialsProvider (credentialsSigner: CredentialsSigner, partyKey: PartyKey, feedKey: FeedKey): CredentialsProvider {
-  return {
-    get: () => {
-      const authMessage = createAuthMessage(
+export const createCredentialsProvider = (credentialsSigner: CredentialsSigner, partyKey: PartyKey, feedKey: FeedKey): CredentialsProvider => ({
+  get: () => {
+    const authMessage = createAuthMessage(
+      credentialsSigner.signer,
+      partyKey,
+      credentialsSigner.getIdentityKey(),
+      credentialsSigner.getDeviceSigningKeys(),
+      feedKey,
+      undefined,
+      createFeedAdmitMessage(
         credentialsSigner.signer,
         partyKey,
-        credentialsSigner.getIdentityKey(),
-        credentialsSigner.getDeviceSigningKeys(),
         feedKey,
-        undefined,
-        createFeedAdmitMessage(
-          credentialsSigner.signer,
-          partyKey,
-          feedKey,
-          [feedKey, credentialsSigner.getDeviceSigningKeys()]
-        )
-      );
-      return Buffer.from(codec.encode(authMessage));
-    }
-  };
-}
+        [feedKey, credentialsSigner.getDeviceSigningKeys()]
+      )
+    );
+    return Buffer.from(codec.encode(authMessage));
+  }
+});

package/src/protocol/halo-recovery-plugin.ts

@@ -12,9 +12,7 @@ import { HaloRecoveryInitiator, InvitationFactory } from '../invitations';
  * Plugin is intended to be used in HALO party swarm.
  *
  */
-export function createHaloRecoveryPlugin (identityKey: PublicKey, invitationFactory: InvitationFactory, peerId: PublicKey) {
-  return new GreetingCommandPlugin(
-    peerId.asBuffer(),
-    HaloRecoveryInitiator.createHaloInvitationClaimHandler(identityKey, invitationFactory)
-  );
-}
+export const createHaloRecoveryPlugin = (identityKey: PublicKey, invitationFactory: InvitationFactory, peerId: PublicKey) => new GreetingCommandPlugin(
+  peerId.asBuffer(),
+  HaloRecoveryInitiator.createHaloInvitationClaimHandler(identityKey, invitationFactory)
+);

package/src/protocol/identity-credentials.ts

@@ -25,7 +25,7 @@ export interface IdentityCredentials {
 
 export type IdentityCredentialsProvider = () => IdentityCredentials | undefined
 
-export async function createTestIdentityCredentials (keyring: Keyring): Promise<IdentityCredentials> {
+export const createTestIdentityCredentials = async (keyring: Keyring): Promise<IdentityCredentials> => {
   const identityKey = await keyring.createKeyRecord({ type: KeyType.IDENTITY });
   const deviceKey = await keyring.createKeyRecord({ type: KeyType.DEVICE });
   const feedKey = await keyring.createKeyRecord({ type: KeyType.FEED });
@@ -53,9 +53,9 @@ export async function createTestIdentityCredentials (keyring: Keyring): Promise<
     preferences: undefined,
     contacts: undefined
   };
-}
+};
 
-export async function deriveTestDeviceCredentials (identity: IdentityCredentials): Promise<IdentityCredentials> {
+export const deriveTestDeviceCredentials = async (identity: IdentityCredentials): Promise<IdentityCredentials> => {
   const deviceKey = await identity.keyring.createKeyRecord({ type: KeyType.DEVICE });
   const keyAdmit = createKeyAdmitMessage(identity.keyring, identity.identityKey.publicKey, deviceKey, [identity.identityKey]);
 
@@ -75,4 +75,4 @@ export async function deriveTestDeviceCredentials (identity: IdentityCredentials
       deviceKeyChain
     )
   };
-}
+};

package/src/protocol/offline-invitation-plugin.ts

@@ -11,9 +11,7 @@ import { InvitationFactory, OfflineInvitationClaimer } from '../invitations';
  * Creates network protocol plugin that allows peers to claim offline invitations.
  * Plugin is intended to be used in data-party swarms.
  */
-export function createOfflineInvitationPlugin (invitationFactory: InvitationFactory, peerId: PublicKey) {
-  return new GreetingCommandPlugin(
-    peerId.asBuffer(),
-    OfflineInvitationClaimer.createOfflineInvitationClaimHandler(invitationFactory)
-  );
-}
+export const createOfflineInvitationPlugin = (invitationFactory: InvitationFactory, peerId: PublicKey) => new GreetingCommandPlugin(
+  peerId.asBuffer(),
+  OfflineInvitationClaimer.createOfflineInvitationClaimHandler(invitationFactory)
+);

package/src/protocol/party-protocol-factory.ts

@@ -4,17 +4,13 @@
 
 import debug from 'debug';
 
-import { synchronized } from '@dxos/async';
 import { discoveryKey, keyToString, PublicKey } from '@dxos/crypto';
-import { FeedKey, PartyKey } from '@dxos/echo-protocol';
-import type { HypercoreFeed } from '@dxos/feed-store';
+import { PartyKey } from '@dxos/echo-protocol';
 import { Protocol } from '@dxos/mesh-protocol';
 import { MMSTTopology, NetworkManager, Plugin } from '@dxos/network-manager';
 import { PresencePlugin } from '@dxos/protocol-plugin-presence';
-import { Replicator } from '@dxos/protocol-plugin-replicator';
 
 import { CredentialsProvider } from '.';
-import { PartyFeedProvider } from '../pipeline/party-feed-provider';
 
 const log = debug('dxos:echo-db:party-protocol-factory');
 
@@ -23,21 +19,15 @@ const log = debug('dxos:echo-db:party-protocol-factory');
  */
 export class PartyProtocolFactory {
   private readonly _presencePlugin = new PresencePlugin(this._peerId.asBuffer());
-  private readonly _replicatorProtocolPluginFactory: ReplicatorProtocolPluginFactory;
 
   private _started = false;
 
   constructor (
     private readonly _partyKey: PartyKey,
     private readonly _networkManager: NetworkManager,
-    private readonly _feedProvider: PartyFeedProvider,
     private readonly _peerId: PublicKey,
     private readonly _credentials: CredentialsProvider
-  ) {
-    // Replication.
-    this._replicatorProtocolPluginFactory =
-      new ReplicatorProtocolPluginFactory(this._feedProvider);
-  }
+  ) {}
 
   async start (plugins: Plugin[]) {
     if (this._started) {
@@ -73,9 +63,8 @@ export class PartyProtocolFactory {
     await this._networkManager.leaveProtocolSwarm(this._partyKey);
   }
 
-  private _createProtocol (channel: any, opts: {initiator: boolean}, extraPlugins: Plugin[]) {
+  private _createProtocol (channel: any, opts: { initiator: boolean }, extraPlugins: Plugin[]) {
     const plugins: Plugin[] = [
-      ...this._replicatorProtocolPluginFactory.createPlugins(),
       ...extraPlugins,
       this._presencePlugin
     ];
@@ -118,45 +107,3 @@ export class PartyProtocolFactory {
     return protocol;
   }
 }
-
-/**
- * Creates the protocol plugin for feed replication.
- */
-export class ReplicatorProtocolPluginFactory {
-  constructor (
-    private readonly _feedProvider: PartyFeedProvider
-  ) {}
-
-  createPlugins () {
-    return [
-      new Replicator({
-        load: async () => {
-          const feeds = this._feedProvider.getFeeds();
-          log(`Loading feeds: ${feeds.map(feed => keyToString(feed.key))}`);
-          return feeds.map((feed) => ({ discoveryKey: feed.feed.discoveryKey }));
-        },
-
-        subscribe: (addFeedToReplicatedSet: (feed: any) => void) => {
-          return this._feedProvider.feedOpened.on(async (feed) => {
-            log(`Adding feed: ${feed.key.toHex()}`);
-            addFeedToReplicatedSet({ discoveryKey: feed.feed.discoveryKey });
-          });
-        },
-
-        replicate: async (remoteFeeds, info) => {
-          // We can ignore remoteFeeds entirely, since the set of feeds we want to replicate is dictated by the Party.
-          // TODO(telackey): Why are we opening feeds? Necessary or belt/braces thinking, or because open party does it?
-          const feeds = this._feedProvider.getFeeds();
-          log(`Replicating: peerId=${info.session}; feeds=${feeds.map(feed => feed.key.toHex())}`);
-          return feeds.map(feed => feed.feed);
-        }
-      })
-    ];
-  }
-
-  @synchronized
-  private async _openFeed (key: FeedKey): Promise<HypercoreFeed> {
-    const descriptor = await this._feedProvider.createOrOpenReadOnlyFeed(key);
-    return descriptor.feed;
-  }
-}

package/src/protocol/replicator-plugin.ts

@@ -0,0 +1,37 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import debug from 'debug';
+
+import { keyToString } from '@dxos/crypto';
+import { Replicator } from '@dxos/protocol-plugin-replicator';
+
+import { PartyFeedProvider } from '../pipeline';
+
+const log = debug('dxos:echo-db:protocol:replicator');
+
+/**
+ * Creates the protocol plugin for feed replication.
+ */
+export const createReplicatorPlugin = (feedProvider: PartyFeedProvider) =>
+  new Replicator({
+    load: async () => {
+      const feeds = feedProvider.getFeeds();
+      log(`Loading feeds: ${feeds.map(feed => keyToString(feed.key))}`);
+      return feeds.map((feed) => ({ discoveryKey: feed.feed.discoveryKey }));
+    },
+
+    subscribe: (addFeedToReplicatedSet: (feed: any) => void) => feedProvider.feedOpened.on(async (feed) => {
+      log(`Adding feed: ${feed.key.toHex()}`);
+      addFeedToReplicatedSet({ discoveryKey: feed.feed.discoveryKey });
+    }),
+
+    replicate: async (remoteFeeds, info) => {
+      // We can ignore remoteFeeds entirely, since the set of feeds we want to replicate is dictated by the Party.
+      // TODO(telackey): Why are we opening feeds? Necessary or belt/braces thinking, or because open party does it?
+      const feeds = feedProvider.getFeeds();
+      log(`Replicating: peerId=${info.session}; feeds=${feeds.map(feed => feed.key.toHex())}`);
+      return feeds.map(feed => feed.feed);
+    }
+  });

package/src/snapshots/snapshot-generator.ts

@@ -10,20 +10,16 @@ import { SnapshotStore } from './snapshot-store';
 
 const log = debug('dxos:snapshot-generator');
 
-export function createAutomaticSnapshots (
-  party: PartyCore, clock: TimeframeClock, store: SnapshotStore, interval: number
-) {
-  return clock.update.on(async timeframe => {
-    const totalMessages = timeframe.totalMessages();
-    if (totalMessages > 0 && totalMessages % interval === 0) {
-      log(`Saving snapshot of ${party.key.humanize()}...`);
-      try {
-        const snapshot = party.createSnapshot();
-        await store.save(snapshot);
-      } catch (err: any) {
-        console.error('Failed to save snapshot');
-        console.error(err);
-      }
+export const createAutomaticSnapshots = (party: PartyCore, clock: TimeframeClock, store: SnapshotStore, interval: number) => clock.update.on(async timeframe => {
+  const totalMessages = timeframe.totalMessages();
+  if (totalMessages > 0 && totalMessages % interval === 0) {
+    log(`Saving snapshot of ${party.key.humanize()}...`);
+    try {
+      const snapshot = party.createSnapshot();
+      await store.save(snapshot);
+    } catch (err: any) {
+      console.error('Failed to save snapshot');
+      console.error(err);
     }
-  });
-}
+  }
+});

package/src/snapshots/snapshot-store.test.ts

@@ -15,7 +15,7 @@ const createPublicKey = () => PublicKey.from(createKeyPair().publicKey);
 
 describe('SnapshotStore', () => {
   test('in-memory', async () => {
-    const store = new SnapshotStore(createStorage('snapshots', StorageType.RAM));
+    const store = new SnapshotStore(createStorage('', StorageType.RAM).directory('snapshots'));
 
     const key1 = createPublicKey();
     const key2 = createPublicKey();

package/src/snapshots/snapshot-store.ts

@@ -7,10 +7,9 @@ import debug from 'debug';
 
 import { keyToString } from '@dxos/crypto';
 import { schema, PartyKey, PartySnapshot } from '@dxos/echo-protocol';
-import { Storage } from '@dxos/random-access-multi-storage';
+import { Directory } from '@dxos/random-access-multi-storage';
 
 const log = debug('dxos:snapshot-store');
-
 /**
  * Stores party snapshots. Takes any `random-access-storage` compatible backend.
  *
@@ -18,11 +17,11 @@ const log = debug('dxos:snapshot-store');
  */
 export class SnapshotStore {
   constructor (
-    private readonly _storage: Storage
+    private readonly _directory: Directory
   ) {}
 
   async load (partyKey: PartyKey): Promise<PartySnapshot | undefined> {
-    const file = this._storage.createOrOpen(partyKey.toHex());
+    const file = this._directory.createOrOpen(partyKey.toHex());
 
     try {
       const { size } = await file.stat();
@@ -45,7 +44,7 @@ export class SnapshotStore {
 
   async save (snapshot: PartySnapshot) {
     assert(snapshot.partyKey);
-    const file = this._storage.createOrOpen(keyToString(snapshot.partyKey), { truncate: true, size: 0 });
+    const file = this._directory.createOrOpen(keyToString(snapshot.partyKey), { truncate: true, size: 0 });
 
     try {
       const data = schema.getCodecForType('dxos.echo.snapshot.PartySnapshot').encode(snapshot);
@@ -60,6 +59,6 @@ export class SnapshotStore {
    */
   async clear () {
     log('Clearing all snapshots..');
-    await this._storage.destroy();
+    await this._directory.destroy();
   }
 }

package/src/testing/benchmark.test.ts

@@ -0,0 +1,30 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { ObjectModel } from '@dxos/object-model';
+
+import { ECHO } from '../echo';
+
+const ITEMS = 100;
+const MUTATIONS = 1000;
+
+it.skip('Database benchmark', async () => {
+  const echo = new ECHO({ snapshots: false });
+  await echo.open();
+  await echo.halo.createProfile();
+  const party = await echo.createParty();
+
+  const start = Date.now();
+  for (let i = 0; i < ITEMS; i++) {
+    const item = await party.database.createItem({ model: ObjectModel, type: 'test:item' });
+
+    for (let j = 0; j < MUTATIONS; j++) {
+      await item.model.set(`key${j % 100}`, `value-${j}`);
+    }
+
+    console.log(`${i * MUTATIONS}/${ITEMS * MUTATIONS} ${((Date.now() - start) / ((i + 1) * MUTATIONS / 1000)).toFixed(1)} µs/mut ${((i + 1) * MUTATIONS / (Date.now() - start) * 1000).toFixed(1)} mut/s`);
+  }
+
+  await echo.close();
+});

package/src/testing/testing-factories.ts

@@ -22,10 +22,10 @@ export type TestPeer = Awaited<ReturnType<typeof createTestInstance>>;
 
 export type WithTestMeta<T> = T & { testMeta: TestPeer }
 
-function addTestMeta<T> (obj: T, meta: TestPeer): WithTestMeta<T> {
+const addTestMeta = <T>(obj: T, meta: TestPeer): WithTestMeta<T> => {
   (obj as any).testMeta = meta;
   return obj as any;
-}
+};
 
 /**
  * Creates a number of test ECHO instances and a party that's shared between all of them.