@dxos/echo-db 2.33.0 → 2.33.1-dev.17ad8706

package/src/parties/party-manager.test.ts

@@ -29,7 +29,7 @@ import { createWritableFeedStream, FeedStore } from '@dxos/feed-store';
 import { ModelFactory } from '@dxos/model-factory';
 import { NetworkManager } from '@dxos/network-manager';
 import { ObjectModel } from '@dxos/object-model';
-import { createStorage, STORAGE_RAM } from '@dxos/random-access-multi-storage';
+import { createStorage, StorageType } from '@dxos/random-access-multi-storage';
 import { afterTest } from '@dxos/testutils';
 
 import { Item } from '../api';
@@ -58,7 +58,7 @@ const log = debug('dxos:echo:parties:party-manager:test');
 const setup = async (open = true, createIdentity = true) => {
   const keyring = new Keyring();
   const metadataStore = new MetadataStore(createRamStorage());
-  const feedStore = new FeedStore(createStorage('', STORAGE_RAM), { valueEncoding: codec });
+  const feedStore = new FeedStore(createStorage('', StorageType.RAM), { valueEncoding: codec });
 
   let seedPhrase;
   if (createIdentity) {
@@ -73,7 +73,7 @@ const setup = async (open = true, createIdentity = true) => {
     assert(keyring.keys.length === 1);
   }
 
-  const snapshotStore = new SnapshotStore(createStorage('', STORAGE_RAM));
+  const snapshotStore = new SnapshotStore(createStorage('', StorageType.RAM));
   const modelFactory = new ModelFactory().registerModel(ObjectModel);
   const networkManager = new NetworkManager();
   const feedProviderFactory = (partyKey: PublicKey) => new PartyFeedProvider(metadataStore, keyring, feedStore, partyKey);
@@ -179,7 +179,7 @@ describe('Party manager', () => {
   });
 
   test('Create from cold start', async () => {
-    const storage = createStorage('', STORAGE_RAM);
+    const storage = createStorage('', StorageType.RAM);
     const feedStore = new FeedStore(storage, { valueEncoding: codec });
 
     const keyring = new Keyring();
@@ -189,7 +189,7 @@ describe('Party manager', () => {
     await keyring.createKeyRecord({ type: KeyType.DEVICE });
 
     const modelFactory = new ModelFactory().registerModel(ObjectModel);
-    const snapshotStore = new SnapshotStore(createStorage('', STORAGE_RAM));
+    const snapshotStore = new SnapshotStore(createStorage('', StorageType.RAM));
     const networkManager = new NetworkManager();
     const feedProviderFactory = (partyKey: PublicKey) => new PartyFeedProvider(metadataStore, keyring, feedStore, partyKey);
     const partyFactory: PartyFactory = new PartyFactory(() => identityManager.identity, networkManager, modelFactory, snapshotStore, feedProviderFactory);

package/src/pipeline/party-feed-provider.ts

@@ -5,10 +5,12 @@
 import assert from 'assert';
 import debug from 'debug';
 
+import { Event } from '@dxos/async';
 import { Keyring, KeyType } from '@dxos/credentials';
 import { PublicKey } from '@dxos/crypto';
 import { FeedStoreIterator, MessageSelector, Timeframe } from '@dxos/echo-protocol';
 import { FeedDescriptor, FeedStore } from '@dxos/feed-store';
+import { ComplexMap } from '@dxos/util';
 
 import { MetadataStore } from '../metadata';
 
@@ -16,6 +18,9 @@ const STALL_TIMEOUT = 1000;
 const warn = debug('dxos:echo-db:party-feed-provider:warn');
 
 export class PartyFeedProvider {
+  private readonly _feeds = new ComplexMap<PublicKey, FeedDescriptor>(x => x.toHex())
+  readonly feedOpened = new Event<FeedDescriptor>();
+
   constructor (
     private readonly _metadataStore: MetadataStore,
     private readonly _keyring: Keyring,
@@ -23,14 +28,13 @@ export class PartyFeedProvider {
     private readonly _partyKey: PublicKey
   ) {}
 
-  // TODO(dmaretskyi): Consider refactoring this to have write feed stored separeately in metadata.
+  getFeeds (): FeedDescriptor[] {
+    return Array.from(this._feeds.values());
+  }
+
   async createOrOpenWritableFeed () {
     const partyMetadata = this._metadataStore.getParty(this._partyKey);
-    if (!partyMetadata) {
-      return this._createReadWriteFeed();
-    }
-
-    if (!partyMetadata.dataFeedKey) {
+    if (!partyMetadata?.dataFeedKey) {
       return this._createReadWriteFeed();
     }
 
@@ -39,22 +43,42 @@ export class PartyFeedProvider {
       return this._createReadWriteFeed();
     }
 
+    if (this._feeds.has(fullKey.publicKey)) {
+      return this._feeds.get(fullKey.publicKey)!;
+    }
+
     const feed = await this._feedStore.openReadWriteFeed(fullKey.publicKey, fullKey.secretKey);
-    const feedKey = this._keyring.getKey(feed.key);
-    assert(feedKey, 'Feed key not found');
+    this._feeds.set(fullKey.publicKey, feed);
+    this.feedOpened.emit(feed);
     return feed;
   }
 
-  getFeedKeys () {
-    return this._metadataStore.getParty(this._partyKey)?.feedKeys ?? [];
+  async openKnownFeeds () {
+    for (const feedKey of this._metadataStore.getParty(this._partyKey)?.feedKeys ?? []) {
+      if (!this._feeds.has(feedKey)) {
+        const fullKey = this._keyring.getFullKey(feedKey);
+        const feed = fullKey?.secretKey
+          ? await this._feedStore.openReadWriteFeed(fullKey.publicKey, fullKey.secretKey)
+          : await this._feedStore.openReadOnlyFeed(feedKey);
+        this._feeds.set(feedKey, feed);
+        this.feedOpened.emit(feed);
+      }
+    }
   }
 
   async createOrOpenReadOnlyFeed (feedKey: PublicKey): Promise<FeedDescriptor> {
+    if (this._feeds.has(feedKey)) {
+      return this._feeds.get(feedKey)!;
+    }
+
     await this._metadataStore.addPartyFeed(this._partyKey, feedKey);
     if (!this._keyring.hasKey(feedKey)) {
       await this._keyring.addPublicKey({ type: KeyType.FEED, publicKey: feedKey });
     }
-    return this._feedStore.openReadOnlyFeed(feedKey);
+    const feed = await this._feedStore.openReadOnlyFeed(feedKey);
+    this._feeds.set(feedKey, feed);
+    this.feedOpened.emit(feed);
+    return feed;
   }
 
   private async _createReadWriteFeed () {
@@ -63,16 +87,18 @@ export class PartyFeedProvider {
     assert(fullKey && fullKey.secretKey);
     await this._metadataStore.setDataFeed(this._partyKey, fullKey.publicKey);
     const feed = await this._feedStore.openReadWriteFeed(fullKey.publicKey, fullKey.secretKey);
+    this._feeds.set(fullKey.publicKey, feed);
+    this.feedOpened.emit(feed);
     return feed;
   }
 
   async createIterator (messageSelector: MessageSelector, initialTimeframe?: Timeframe) {
     const iterator = new FeedStoreIterator(() => true, messageSelector, initialTimeframe ?? new Timeframe());
-    for (const feedKey of this.getFeedKeys()) {
-      iterator.addFeedDescriptor(await this.createOrOpenReadOnlyFeed(feedKey));
+    for (const feed of this._feeds.values()) {
+      iterator.addFeedDescriptor(feed);
     }
 
-    this._feedStore.feedOpenedEvent.on((descriptor) => {
+    this.feedOpened.on((descriptor) => {
       if (this._metadataStore.getParty(this._partyKey)?.feedKeys?.find(feedKey => feedKey.equals(descriptor.key))) {
         iterator.addFeedDescriptor(descriptor);
       }

package/src/pipeline/party-processor.test.ts

@@ -36,7 +36,7 @@ describe('party-processor', () => {
 
     const message: IHaloStream = {
       meta: {
-        feedKey: feedKey.publicKey.asUint8Array(),
+        feedKey: feedKey.publicKey,
         seq: 0
         // TODO(telackey): Should ownership data go here?
       },
@@ -67,7 +67,7 @@ describe('party-processor', () => {
 
     const genesisMessage: IHaloStream = {
       meta: {
-        feedKey: feedKey.publicKey.asUint8Array(),
+        feedKey: feedKey.publicKey,
         seq: 0
         // TODO(telackey): Should ownership data go here?
       },
@@ -78,7 +78,7 @@ describe('party-processor', () => {
     const feedKey2 = await keyring.createKeyRecord({ type: KeyType.FEED });
     const feedAdmit: IHaloStream = {
       meta: {
-        feedKey: feedKey.publicKey.asUint8Array(),
+        feedKey: feedKey.publicKey,
         seq: 0
         // TODO(telackey): Should ownership data go here?
       },
@@ -106,7 +106,7 @@ describe('party-processor', () => {
 
     const genesisMessage: IHaloStream = {
       meta: {
-        feedKey: feedKey.publicKey.asUint8Array(),
+        feedKey: feedKey.publicKey,
         seq: 0
         // TODO(telackey): Should ownership data go here?
       },
@@ -115,7 +115,7 @@ describe('party-processor', () => {
     await partyProcessor.processMessage(genesisMessage);
     const feedAdmit: IHaloStream = {
       meta: {
-        feedKey: feedKey.publicKey.asUint8Array(),
+        feedKey: feedKey.publicKey,
         seq: 0
         // TODO(telackey): Should ownership data go here?
       },
@@ -129,7 +129,7 @@ describe('party-processor', () => {
 
     const keyAdmit: IHaloStream = {
       meta: {
-        feedKey: feedKey.publicKey.asUint8Array(),
+        feedKey: feedKey.publicKey,
         seq: 1
       },
       data: createEnvelopeMessage(keyring, partyKey.publicKey,
@@ -141,7 +141,7 @@ describe('party-processor', () => {
 
     const feedAdmit2: IHaloStream = {
       meta: {
-        feedKey: feedKey.publicKey.asUint8Array(),
+        feedKey: feedKey.publicKey,
         seq: 1
         // TODO(telackey): Should ownership data go here?
       },
@@ -170,7 +170,7 @@ describe('party-processor', () => {
     const partyProcessor = new PartyProcessor(partyKey.publicKey);
     expect(partyProcessor.partyKey).toBeTruthy();
 
-    const meta = (seq: number) => ({ feedKey: feedKey.publicKey.asUint8Array(), seq });
+    const meta = (seq: number) => ({ feedKey: feedKey.publicKey, seq });
 
     await partyProcessor.processMessage({
       meta: meta(0),

package/src/pipeline/party-processor.ts

@@ -7,11 +7,9 @@ import debug from 'debug';
 
 import { Event } from '@dxos/async';
 import {
-  Authenticator,
   KeyHint,
   KeyRecord,
   PartyState,
-  PartyAuthenticator,
   Message as HaloMessage,
   IdentityEventType,
   PartyEventType
@@ -32,11 +30,10 @@ export interface FeedSetProvider {
  */
 export class PartyProcessor {
   private readonly _state: PartyState;
-  private readonly _authenticator: Authenticator;
 
   private _outboundHaloStream: FeedWriter<HaloMessage> | undefined;
 
-  protected readonly _feedAdded = new Event<FeedKey>()
+  readonly feedAdded = new Event<FeedKey>()
 
   public readonly keyOrInfoAdded = new Event<PublicKey>();
 
@@ -49,21 +46,14 @@ export class PartyProcessor {
     private readonly _partyKey: PartyKey
   ) {
     this._state = new PartyState(this._partyKey);
-    this._authenticator = new PartyAuthenticator(this._state);
-
-    /* TODO(telackey): `@dxos/credentials` was only half converted to TS. In its current state, the KeyRecord type
-     * is not exported, and the PartyStateMachine being used is not properly understood as an EventEmitter by TS.
-     * Casting to 'any' is a workaround for the compiler, but the fix is fully to convert @dxos/credentials to TS.
-     */
-    const state = this._state as any;
 
     // TODO(marik-d): Use `Event.wrap` here.
-    state.on(PartyEventType.ADMIT_FEED, (keyRecord: any) => {
+    this._state.on(PartyEventType.ADMIT_FEED, (keyRecord: any) => {
       log(`Feed key admitted ${keyRecord.publicKey.toHex()}`);
-      this._feedAdded.emit(keyRecord.publicKey);
+      this.feedAdded.emit(keyRecord.publicKey);
     });
-    state.on(PartyEventType.ADMIT_KEY, (keyRecord: KeyRecord) => this.keyOrInfoAdded.emit(keyRecord.publicKey));
-    state.on(IdentityEventType.UPDATE_IDENTITY, (publicKey: PublicKey) => this.keyOrInfoAdded.emit(publicKey));
+    this._state.on(PartyEventType.ADMIT_KEY, (keyRecord: KeyRecord) => this.keyOrInfoAdded.emit(keyRecord.publicKey));
+    this._state.on(IdentityEventType.UPDATE_IDENTITY, (publicKey: PublicKey) => this.keyOrInfoAdded.emit(publicKey));
   }
 
   get partyKey () {
@@ -90,8 +80,8 @@ export class PartyProcessor {
     return this._state.credentialMessages.size === 0;
   }
 
-  get authenticator () {
-    return this._authenticator;
+  get state () {
+    return this._state;
   }
 
   isFeedAdmitted (feedKey: FeedKey) {
@@ -122,7 +112,7 @@ export class PartyProcessor {
   getActiveFeedSet (): FeedSetProvider {
     return {
       get: () => this.feedKeys,
-      added: this._feedAdded
+      added: this.feedAdded
     };
   }
 

package/src/pipeline/party-protocol-factory.ts

@@ -144,7 +144,7 @@ export class PartyProtocolFactory {
 /**
  * Creates the protocol plugin for feed replication.
  */
-class ReplicatorProtocolPluginFactory {
+export class ReplicatorProtocolPluginFactory {
   constructor (
     private readonly _feedProvider: PartyFeedProvider,
     private readonly _activeFeeds: FeedSetProvider

package/src/pipeline/pipeline.test.ts

@@ -12,7 +12,7 @@ import { createId, createKeyPair, PublicKey } from '@dxos/crypto';
 import { codec, createFeedWriter, FeedSelector, FeedStoreIterator, IEchoStream, Timeframe } from '@dxos/echo-protocol';
 import { FeedStore, createWritableFeedStream, createWritable, WritableArray } from '@dxos/feed-store';
 import { createSetPropertyMutation } from '@dxos/model-factory';
-import { createStorage, STORAGE_RAM } from '@dxos/random-access-multi-storage';
+import { createStorage, StorageType } from '@dxos/random-access-multi-storage';
 import { jsonReplacer } from '@dxos/util';
 
 import { TimeframeClock } from '../database';
@@ -24,7 +24,7 @@ const log = debug('dxos:echo:pipeline:test');
 // TODO(burdon): Test read-only.
 describe('pipeline', () => {
   test('streams', async () => {
-    const storage = createStorage('', STORAGE_RAM);
+    const storage = createStorage('', StorageType.RAM);
     const feedStore = new FeedStore(storage, { valueEncoding: codec });
     const feedKeys: Uint8Array[] = [];
     const feedSelector: FeedSelector = descriptor => !!feedKeys.find(key => descriptor.key.equals(key));
@@ -52,7 +52,7 @@ describe('pipeline', () => {
     await partyProcessor.processMessage({
       data: createPartyGenesisMessage(keyring, partyKey, feedKey.publicKey, identityKey),
       meta: {
-        feedKey: feedKey.publicKey.asBuffer(),
+        feedKey: feedKey.publicKey,
         seq: 0
       }
     });
@@ -86,7 +86,7 @@ describe('pipeline', () => {
   });
 
   test('writing', async () => {
-    const storage = createStorage('', STORAGE_RAM);
+    const storage = createStorage('', StorageType.RAM);
     const feedStore = new FeedStore(storage, { valueEncoding: codec });
     const feedReadStream = new FeedStoreIterator(() => true, () => 0, new Timeframe());
 

package/src/pipeline/pipeline.ts

@@ -141,7 +141,7 @@ export class Pipeline {
                 meta: {
                   seq: block.seq,
                   feedKey: block.key,
-                  memberKey: memberKey.asUint8Array(),
+                  memberKey,
                   timeframe: message.echo.timeframe ?? new Timeframe()
                 },
                 data: message.echo

package/src/util/persistant-ram-storage.ts

@@ -2,7 +2,7 @@
 // Copyright 2020 DXOS.org
 //
 
-import { createStorage, IFile, IStorage } from '@dxos/random-access-multi-storage';
+import { StorageType, createStorage, File, IStorage } from '@dxos/random-access-multi-storage';
 
 /**
  * A wrapper around RAM storage that preserves file data when closing and re-opening files.
@@ -10,8 +10,8 @@ import { createStorage, IFile, IStorage } from '@dxos/random-access-multi-storag
 // TODO(burdon): Factor out?
 export function createRamStorage (): IStorage {
   const root = 'snapshots';
-  const storage = createStorage(root, 'ram');
-  const files = new Map<string, IFile>();
+  const storage = createStorage(root, StorageType.RAM);
+  const files = new Map<string, File>();
 
   const fn: any = {};