@dxos/echo-db 2.33.1-dev.0a5ae2e2 → 2.33.1-dev.25337170

package/src/parties/party-core.test.ts

@@ -5,24 +5,26 @@
 import expect from 'expect';
 import { it as test } from 'mocha';
 
+import { promiseTimeout } from '@dxos/async';
 import { createFeedAdmitMessage, createPartyGenesisMessage, Keyring, KeyType } from '@dxos/credentials';
-import { PublicKey } from '@dxos/crypto';
-import { codec } from '@dxos/echo-protocol';
+import { createId, PublicKey } from '@dxos/crypto';
+import { codec, Timeframe } from '@dxos/echo-protocol';
 import { FeedStore } from '@dxos/feed-store';
+import { createTestProtocolPair } from '@dxos/mesh-protocol';
 import { ModelFactory } from '@dxos/model-factory';
 import { ObjectModel } from '@dxos/object-model';
-import { createStorage, STORAGE_RAM } from '@dxos/random-access-multi-storage';
+import { createStorage, StorageType } from '@dxos/random-access-multi-storage';
 import { afterTest } from '@dxos/testutils';
 
 import { MetadataStore } from '../metadata';
-import { PartyFeedProvider } from '../pipeline';
+import { PartyFeedProvider, ReplicatorProtocolPluginFactory } from '../pipeline';
 import { SnapshotStore } from '../snapshots';
 import { createRamStorage } from '../util';
 import { PartyCore } from './party-core';
 
 describe('PartyCore', () => {
   const setup = async () => {
-    const storage = createStorage('', STORAGE_RAM);
+    const storage = createStorage('', StorageType.RAM);
     const feedStore = new FeedStore(storage, { valueEncoding: codec });
     afterTest(async () => feedStore.close());
 
@@ -31,7 +33,7 @@ describe('PartyCore', () => {
     const metadataStore = new MetadataStore(createRamStorage());
 
     const modelFactory = new ModelFactory().registerModel(ObjectModel);
-    const snapshotStore = new SnapshotStore(createStorage('', STORAGE_RAM));
+    const snapshotStore = new SnapshotStore(createStorage('', StorageType.RAM));
 
     const partyKey = await keyring.createKeyRecord({ type: KeyType.PARTY });
 
@@ -65,10 +67,7 @@ describe('PartyCore', () => {
       [partyKey]
     ));
 
-    // The Party key is an inception key; its SecretKey must be destroyed once the Party has been created.
-    await keyring.deleteSecretKey(partyKey);
-
-    return { party, feedKey: feed.key, feed, feedStore };
+    return { party, feedKey: feed.key, feed, feedStore, partyKey, keyring, partyFeedProvider };
   };
 
   test('create & have the feed key admitted', async () => {
@@ -115,4 +114,228 @@ describe('PartyCore', () => {
       expect(parent.children).toContain(child);
     }
   });
+
+  test('feed admit message triggers new feed to be opened', async () => {
+    const { party, partyKey, keyring, partyFeedProvider, feedStore } = await setup();
+
+    const feedKey = await keyring.createKeyRecord({ type: KeyType.FEED });
+
+    const eventFired = feedStore.feedOpenedEvent.waitForCount(1);
+    await party.processor.writeHaloMessage(createFeedAdmitMessage(
+      keyring,
+      party.key,
+      feedKey.publicKey,
+      [partyKey]
+    ));
+    await promiseTimeout(eventFired, 1000, new Error('timeout'));
+    expect(partyFeedProvider.getFeeds().find(k => k.key.equals(feedKey.publicKey))).toBeTruthy();
+  });
+
+  test('opens feed from hints', async () => {
+    const storage = createStorage('', StorageType.RAM);
+    const feedStore = new FeedStore(storage, { valueEncoding: codec });
+    afterTest(async () => feedStore.close());
+
+    const keyring = new Keyring();
+
+    const metadataStore = new MetadataStore(createRamStorage());
+
+    const modelFactory = new ModelFactory().registerModel(ObjectModel);
+    const snapshotStore = new SnapshotStore(createStorage('', StorageType.RAM));
+
+    const partyKey = await keyring.createKeyRecord({ type: KeyType.PARTY });
+
+    const partyFeedProvider = new PartyFeedProvider(metadataStore, keyring, feedStore, partyKey.publicKey);
+
+    const otherFeedKey = PublicKey.random();
+
+    const party = new PartyCore(
+      partyKey.publicKey,
+      partyFeedProvider,
+      modelFactory,
+      snapshotStore,
+      PublicKey.random()
+    );
+
+    await partyFeedProvider.createOrOpenWritableFeed();
+
+    const feedOpened = feedStore.feedOpenedEvent.waitForCount(1);
+
+    await party.open([{ type: KeyType.FEED, publicKey: otherFeedKey }]);
+    afterTest(async () => party.close());
+
+    await feedOpened;
+
+    expect(partyFeedProvider.getFeeds().some(k => k.key.equals(otherFeedKey))).toEqual(true);
+  });
+
+  test('manually create item', async () => {
+    const { party, partyFeedProvider } = await setup();
+    await party.open();
+
+    const feed = await partyFeedProvider.createOrOpenWritableFeed();
+
+    const itemId = createId();
+    await feed.feed.append({
+      echo: {
+        itemId,
+        genesis: {
+          itemType: 'dxos:example',
+          modelType: ObjectModel.meta.type
+        },
+        timeframe: new Timeframe()
+      }
+    });
+
+    await promiseTimeout(party.database.waitForItem({ id: itemId }), 1000, new Error('timeout'));
+  });
+
+  test('admit a second feed to the party', async () => {
+    const { party, keyring, partyKey, feedStore } = await setup();
+    await party.open();
+
+    const feedKey = await keyring.createKeyRecord({ type: KeyType.FEED });
+    const fullKey = keyring.getFullKey(feedKey.publicKey);
+    const feed2 = await feedStore.openReadWriteFeed(fullKey!.publicKey, fullKey!.secretKey!);
+
+    await party.processor.writeHaloMessage(createFeedAdmitMessage(
+      keyring,
+      party.key,
+      feed2.key,
+      [partyKey]
+    ));
+
+    const itemId = createId();
+    await feed2.append({
+      echo: {
+        itemId,
+        genesis: {
+          itemType: 'dxos:example',
+          modelType: ObjectModel.meta.type
+        },
+        timeframe: new Timeframe()
+      }
+    });
+
+    await promiseTimeout(party.database.waitForItem({ id: itemId }), 1000, new Error('timeout'));
+  });
+
+  test('admit feed and then open it', async () => {
+    const { party, keyring, partyKey, feedStore } = await setup();
+    await party.open();
+
+    const feedKey = await keyring.createKeyRecord({ type: KeyType.FEED });
+    const fullKey = keyring.getFullKey(feedKey.publicKey);
+
+    await party.processor.writeHaloMessage(createFeedAdmitMessage(
+      keyring,
+      party.key,
+      feedKey.publicKey,
+      [partyKey]
+    ));
+
+    const feed2 = await feedStore.openReadWriteFeed(fullKey!.publicKey, fullKey!.secretKey!);
+    const itemId = createId();
+    await feed2.append({
+      echo: {
+        itemId,
+        genesis: {
+          itemType: 'dxos:example',
+          modelType: ObjectModel.meta.type
+        },
+        timeframe: new Timeframe()
+      }
+    });
+
+    await promiseTimeout(party.database.waitForItem({ id: itemId }), 1000, new Error('timeout'));
+  });
+
+  test('self-admitting feed with a hint', async () => {
+    const { party, keyring, partyKey, feedStore } = await setup();
+    await party.open();
+
+    const feedKey = await keyring.createKeyRecord({ type: KeyType.FEED });
+    const fullKey = keyring.getFullKey(feedKey.publicKey);
+    const feed2 = await feedStore.openReadWriteFeed(fullKey!.publicKey, fullKey!.secretKey!);
+
+    await party.processor.takeHints([{
+      type: KeyType.FEED,
+      publicKey: feedKey.publicKey
+    }]);
+
+    await feed2.append({
+      halo: createFeedAdmitMessage(
+        keyring,
+        party.key,
+        feedKey.publicKey,
+        [partyKey]
+      )
+    });
+
+    const itemId = createId();
+    await feed2.append({
+      echo: {
+        itemId,
+        genesis: {
+          itemType: 'dxos:example',
+          modelType: ObjectModel.meta.type
+        },
+        timeframe: new Timeframe()
+      }
+    });
+
+    await promiseTimeout(party.database.waitForItem({ id: itemId }), 1000, new Error('timeout'));
+  });
+
+  test('two instances replicating', async () => {
+    const peer1 = await setup();
+
+    const storage = createStorage('', StorageType.RAM);
+    const feedStore = new FeedStore(storage, { valueEncoding: codec });
+    afterTest(async () => feedStore.close());
+
+    const metadataStore = new MetadataStore(createRamStorage());
+
+    const modelFactory = new ModelFactory().registerModel(ObjectModel);
+    const snapshotStore = new SnapshotStore(createStorage('', StorageType.RAM));
+
+    const partyFeedProvider = new PartyFeedProvider(metadataStore, peer1.keyring, feedStore, peer1.party.key);
+
+    const party2 = new PartyCore(
+      peer1.party.key,
+      partyFeedProvider,
+      modelFactory,
+      snapshotStore,
+      PublicKey.random()
+    );
+
+    const feed2 = await partyFeedProvider.createOrOpenWritableFeed();
+
+    await peer1.party.processor.writeHaloMessage(createFeedAdmitMessage(
+      peer1.keyring,
+      peer1.party.key,
+      feed2.key,
+      [peer1.partyKey]
+    ));
+
+    await party2.open([{
+      publicKey: peer1.feedKey,
+      type: KeyType.FEED
+    }]);
+    afterTest(async () => party2.close());
+
+    createTestProtocolPair(
+      new ReplicatorProtocolPluginFactory(
+        peer1.partyFeedProvider,
+        peer1.party.processor.getActiveFeedSet()
+      ).createPlugins().map(r => r.createExtension()),
+      new ReplicatorProtocolPluginFactory(
+        partyFeedProvider,
+        peer1.party.processor.getActiveFeedSet()
+      ).createPlugins().map(r => r.createExtension())
+    );
+
+    const item1 = await peer1.party.database.createItem();
+    await promiseTimeout(party2.database.waitForItem({ id: item1.id }), 1000, new Error('timeout'));
+  });
 });

package/src/parties/party-core.ts

@@ -113,9 +113,14 @@ export class PartyCore {
 
     if (!this._partyProcessor) {
       this._partyProcessor = new PartyProcessor(this._partyKey);
-      if (keyHints.length > 0) {
-        await this._partyProcessor.takeHints(keyHints);
-      }
+    }
+    // Automatically open new admitted feeds.
+    this._subscriptions.push(this._partyProcessor.feedAdded.on(feed => {
+      void this._feedProvider.createOrOpenReadOnlyFeed(feed);
+    }));
+
+    if (keyHints.length > 0) {
+      await this._partyProcessor.takeHints(keyHints);
     }
 
     //

package/src/parties/party-manager.test.ts

@@ -29,7 +29,7 @@ import { createWritableFeedStream, FeedStore } from '@dxos/feed-store';
 import { ModelFactory } from '@dxos/model-factory';
 import { NetworkManager } from '@dxos/network-manager';
 import { ObjectModel } from '@dxos/object-model';
-import { createStorage, STORAGE_RAM } from '@dxos/random-access-multi-storage';
+import { createStorage, StorageType } from '@dxos/random-access-multi-storage';
 import { afterTest } from '@dxos/testutils';
 
 import { Item } from '../api';
@@ -58,7 +58,7 @@ const log = debug('dxos:echo:parties:party-manager:test');
 const setup = async (open = true, createIdentity = true) => {
   const keyring = new Keyring();
   const metadataStore = new MetadataStore(createRamStorage());
-  const feedStore = new FeedStore(createStorage('', STORAGE_RAM), { valueEncoding: codec });
+  const feedStore = new FeedStore(createStorage('', StorageType.RAM), { valueEncoding: codec });
 
   let seedPhrase;
   if (createIdentity) {
@@ -73,7 +73,7 @@ const setup = async (open = true, createIdentity = true) => {
     assert(keyring.keys.length === 1);
   }
 
-  const snapshotStore = new SnapshotStore(createStorage('', STORAGE_RAM));
+  const snapshotStore = new SnapshotStore(createStorage('', StorageType.RAM));
   const modelFactory = new ModelFactory().registerModel(ObjectModel);
   const networkManager = new NetworkManager();
   const feedProviderFactory = (partyKey: PublicKey) => new PartyFeedProvider(metadataStore, keyring, feedStore, partyKey);
@@ -179,7 +179,7 @@ describe('Party manager', () => {
   });
 
   test('Create from cold start', async () => {
-    const storage = createStorage('', STORAGE_RAM);
+    const storage = createStorage('', StorageType.RAM);
     const feedStore = new FeedStore(storage, { valueEncoding: codec });
 
     const keyring = new Keyring();
@@ -189,7 +189,7 @@ describe('Party manager', () => {
     await keyring.createKeyRecord({ type: KeyType.DEVICE });
 
     const modelFactory = new ModelFactory().registerModel(ObjectModel);
-    const snapshotStore = new SnapshotStore(createStorage('', STORAGE_RAM));
+    const snapshotStore = new SnapshotStore(createStorage('', StorageType.RAM));
     const networkManager = new NetworkManager();
     const feedProviderFactory = (partyKey: PublicKey) => new PartyFeedProvider(metadataStore, keyring, feedStore, partyKey);
     const partyFactory: PartyFactory = new PartyFactory(() => identityManager.identity, networkManager, modelFactory, snapshotStore, feedProviderFactory);

package/src/pipeline/party-processor.test.ts

@@ -36,7 +36,7 @@ describe('party-processor', () => {
 
     const message: IHaloStream = {
       meta: {
-        feedKey: feedKey.publicKey.asUint8Array(),
+        feedKey: feedKey.publicKey,
         seq: 0
         // TODO(telackey): Should ownership data go here?
       },
@@ -67,7 +67,7 @@ describe('party-processor', () => {
 
     const genesisMessage: IHaloStream = {
       meta: {
-        feedKey: feedKey.publicKey.asUint8Array(),
+        feedKey: feedKey.publicKey,
         seq: 0
         // TODO(telackey): Should ownership data go here?
       },
@@ -78,7 +78,7 @@ describe('party-processor', () => {
     const feedKey2 = await keyring.createKeyRecord({ type: KeyType.FEED });
     const feedAdmit: IHaloStream = {
       meta: {
-        feedKey: feedKey.publicKey.asUint8Array(),
+        feedKey: feedKey.publicKey,
         seq: 0
         // TODO(telackey): Should ownership data go here?
       },
@@ -106,7 +106,7 @@ describe('party-processor', () => {
 
     const genesisMessage: IHaloStream = {
       meta: {
-        feedKey: feedKey.publicKey.asUint8Array(),
+        feedKey: feedKey.publicKey,
         seq: 0
         // TODO(telackey): Should ownership data go here?
       },
@@ -115,7 +115,7 @@ describe('party-processor', () => {
     await partyProcessor.processMessage(genesisMessage);
     const feedAdmit: IHaloStream = {
       meta: {
-        feedKey: feedKey.publicKey.asUint8Array(),
+        feedKey: feedKey.publicKey,
         seq: 0
         // TODO(telackey): Should ownership data go here?
       },
@@ -129,7 +129,7 @@ describe('party-processor', () => {
 
     const keyAdmit: IHaloStream = {
       meta: {
-        feedKey: feedKey.publicKey.asUint8Array(),
+        feedKey: feedKey.publicKey,
         seq: 1
       },
       data: createEnvelopeMessage(keyring, partyKey.publicKey,
@@ -141,7 +141,7 @@ describe('party-processor', () => {
 
     const feedAdmit2: IHaloStream = {
       meta: {
-        feedKey: feedKey.publicKey.asUint8Array(),
+        feedKey: feedKey.publicKey,
         seq: 1
         // TODO(telackey): Should ownership data go here?
       },
@@ -170,7 +170,7 @@ describe('party-processor', () => {
     const partyProcessor = new PartyProcessor(partyKey.publicKey);
     expect(partyProcessor.partyKey).toBeTruthy();
 
-    const meta = (seq: number) => ({ feedKey: feedKey.publicKey.asUint8Array(), seq });
+    const meta = (seq: number) => ({ feedKey: feedKey.publicKey, seq });
 
     await partyProcessor.processMessage({
       meta: meta(0),

package/src/pipeline/party-processor.ts

@@ -33,7 +33,7 @@ export class PartyProcessor {
 
   private _outboundHaloStream: FeedWriter<HaloMessage> | undefined;
 
-  protected readonly _feedAdded = new Event<FeedKey>()
+  readonly feedAdded = new Event<FeedKey>()
 
   public readonly keyOrInfoAdded = new Event<PublicKey>();
 
@@ -50,7 +50,7 @@ export class PartyProcessor {
     // TODO(marik-d): Use `Event.wrap` here.
     this._state.on(PartyEventType.ADMIT_FEED, (keyRecord: any) => {
       log(`Feed key admitted ${keyRecord.publicKey.toHex()}`);
-      this._feedAdded.emit(keyRecord.publicKey);
+      this.feedAdded.emit(keyRecord.publicKey);
     });
     this._state.on(PartyEventType.ADMIT_KEY, (keyRecord: KeyRecord) => this.keyOrInfoAdded.emit(keyRecord.publicKey));
     this._state.on(IdentityEventType.UPDATE_IDENTITY, (publicKey: PublicKey) => this.keyOrInfoAdded.emit(publicKey));
@@ -112,7 +112,7 @@ export class PartyProcessor {
   getActiveFeedSet (): FeedSetProvider {
     return {
       get: () => this.feedKeys,
-      added: this._feedAdded
+      added: this.feedAdded
     };
   }
 

package/src/pipeline/party-protocol-factory.ts

@@ -144,7 +144,7 @@ export class PartyProtocolFactory {
 /**
  * Creates the protocol plugin for feed replication.
  */
-class ReplicatorProtocolPluginFactory {
+export class ReplicatorProtocolPluginFactory {
   constructor (
     private readonly _feedProvider: PartyFeedProvider,
     private readonly _activeFeeds: FeedSetProvider

package/src/pipeline/pipeline.test.ts

@@ -12,7 +12,7 @@ import { createId, createKeyPair, PublicKey } from '@dxos/crypto';
 import { codec, createFeedWriter, FeedSelector, FeedStoreIterator, IEchoStream, Timeframe } from '@dxos/echo-protocol';
 import { FeedStore, createWritableFeedStream, createWritable, WritableArray } from '@dxos/feed-store';
 import { createSetPropertyMutation } from '@dxos/model-factory';
-import { createStorage, STORAGE_RAM } from '@dxos/random-access-multi-storage';
+import { createStorage, StorageType } from '@dxos/random-access-multi-storage';
 import { jsonReplacer } from '@dxos/util';
 
 import { TimeframeClock } from '../database';
@@ -24,7 +24,7 @@ const log = debug('dxos:echo:pipeline:test');
 // TODO(burdon): Test read-only.
 describe('pipeline', () => {
   test('streams', async () => {
-    const storage = createStorage('', STORAGE_RAM);
+    const storage = createStorage('', StorageType.RAM);
     const feedStore = new FeedStore(storage, { valueEncoding: codec });
     const feedKeys: Uint8Array[] = [];
     const feedSelector: FeedSelector = descriptor => !!feedKeys.find(key => descriptor.key.equals(key));
@@ -52,7 +52,7 @@ describe('pipeline', () => {
     await partyProcessor.processMessage({
       data: createPartyGenesisMessage(keyring, partyKey, feedKey.publicKey, identityKey),
       meta: {
-        feedKey: feedKey.publicKey.asBuffer(),
+        feedKey: feedKey.publicKey,
         seq: 0
       }
     });
@@ -86,7 +86,7 @@ describe('pipeline', () => {
   });
 
   test('writing', async () => {
-    const storage = createStorage('', STORAGE_RAM);
+    const storage = createStorage('', StorageType.RAM);
     const feedStore = new FeedStore(storage, { valueEncoding: codec });
     const feedReadStream = new FeedStoreIterator(() => true, () => 0, new Timeframe());
 

package/src/pipeline/pipeline.ts

@@ -141,7 +141,7 @@ export class Pipeline {
                 meta: {
                   seq: block.seq,
                   feedKey: block.key,
-                  memberKey: memberKey.asUint8Array(),
+                  memberKey,
                   timeframe: message.echo.timeframe ?? new Timeframe()
                 },
                 data: message.echo

package/src/util/persistant-ram-storage.ts

@@ -2,7 +2,7 @@
 // Copyright 2020 DXOS.org
 //
 
-import { createStorage, IFile, IStorage } from '@dxos/random-access-multi-storage';
+import { StorageType, createStorage, File, IStorage } from '@dxos/random-access-multi-storage';
 
 /**
  * A wrapper around RAM storage that preserves file data when closing and re-opening files.
@@ -10,8 +10,8 @@ import { createStorage, IFile, IStorage } from '@dxos/random-access-multi-storag
 // TODO(burdon): Factor out?
 export function createRamStorage (): IStorage {
   const root = 'snapshots';
-  const storage = createStorage(root, 'ram');
-  const files = new Map<string, IFile>();
+  const storage = createStorage(root, StorageType.RAM);
+  const files = new Map<string, File>();
 
   const fn: any = {};