@dxos/echo-pipeline 0.1.52 → 0.1.53-main.01cbc6b

package/src/pipeline/pipeline.ts

@@ -2,7 +2,7 @@
 // Copyright 2022 DXOS.org
 //
 
-import assert from 'node:assert';
+import invariant from 'tiny-invariant';
 
 import { Event, sleep, synchronized, Trigger } from '@dxos/async';
 import { Context, rejectOnDispose } from '@dxos/context';
@@ -36,6 +36,11 @@ export type WaitUntilReachedTargetParams = {
  * External state accessor.
  */
 export class PipelineState {
+  /**
+   * @internal
+   */
+  _ctx = new Context();
+
   // TODO(dmaretskyi): Remove?.
   public readonly timeframeUpdate = this._timeframeClock.update;
 
@@ -51,7 +56,10 @@ export class PipelineState {
    */
   private _targetTimeframe: Timeframe | undefined;
 
-  private _reachedTargetPromise: Promise<void> | undefined;
+  /**
+   * @internal
+   */
+  _reachedTargetPromise: Promise<void> | undefined;
 
   // prettier-ignore
   constructor(
@@ -133,6 +141,7 @@ export class PipelineState {
     if (timeout) {
       return Promise.race([
         rejectOnDispose(ctx),
+        rejectOnDispose(this._ctx),
         this._reachedTargetPromise.then(() => {
           done = true;
         }),
@@ -224,7 +233,7 @@ export class Pipeline implements PipelineAccessor {
   }
 
   get writer(): FeedWriter<FeedMessage.Payload> {
-    assert(this._writer, 'Writer not set.');
+    invariant(this._writer, 'Writer not set.');
     return this._writer;
   }
 
@@ -247,8 +256,8 @@ export class Pipeline implements PipelineAccessor {
   }
 
   setWriteFeed(feed: FeedWrapper<FeedMessage>) {
-    assert(!this._writer, 'Writer already set.');
-    assert(feed.properties.writable, 'Feed must be writable.');
+    invariant(!this._writer, 'Writer already set.');
+    invariant(feed.properties.writable, 'Feed must be writable.');
 
     this._writer = createMappedFeedWriter<FeedMessage.Payload, FeedMessage>(
       (payload: FeedMessage.Payload) => ({
@@ -274,6 +283,9 @@ export class Pipeline implements PipelineAccessor {
     this._isStopping = true;
     await this._feedSetIterator?.close();
     await this._processingTrigger.wait(); // Wait for the in-flight message to be processed.
+    await this._state._ctx.dispose();
+    this._state._ctx = new Context();
+    this._state._reachedTargetPromise = undefined;
     log('stopped');
     this._isStarted = false;
   }
@@ -283,7 +295,7 @@ export class Pipeline implements PipelineAccessor {
    */
   @synchronized
   async setCursor(timeframe: Timeframe) {
-    assert(!this._isStarted || this._isPaused, 'Invalid state.');
+    invariant(!this._isStarted || this._isPaused, 'Invalid state.');
 
     this._state._startTimeframe = timeframe;
     this._timeframeClock.setTimeframe(timeframe);
@@ -305,7 +317,7 @@ export class Pipeline implements PipelineAccessor {
    */
   @synchronized
   async pause() {
-    assert(this._isStarted, 'Pipeline is not open.');
+    invariant(this._isStarted, 'Pipeline is not open.');
     if (this._isPaused) {
       return;
     }
@@ -317,8 +329,8 @@ export class Pipeline implements PipelineAccessor {
 
   @synchronized
   async unpause() {
-    assert(this._isStarted, 'Pipeline is not open.');
-    assert(this._isPaused, 'Pipeline is not paused.');
+    invariant(this._isStarted, 'Pipeline is not open.');
+    invariant(this._isPaused, 'Pipeline is not paused.');
 
     this._pauseTrigger.wake();
     this._isPaused = false;
@@ -329,10 +341,10 @@ export class Pipeline implements PipelineAccessor {
    * Updates the timeframe clock after the message has bee processed.
    */
   async *consume(): AsyncIterable<FeedMessageBlock> {
-    assert(!this._isOpen, 'Pipeline is already being consumed.');
+    invariant(!this._isOpen, 'Pipeline is already being consumed.');
     this._isOpen = true;
 
-    assert(this._feedSetIterator, 'Iterator not initialized.');
+    invariant(this._feedSetIterator, 'Iterator not initialized.');
     let lastFeedSetIterator = this._feedSetIterator;
     let iterable = lastFeedSetIterator[Symbol.asyncIterator]();
 
@@ -341,7 +353,7 @@ export class Pipeline implements PipelineAccessor {
 
       // Iterator might have been changed while we were waiting for the processing to complete.
       if (lastFeedSetIterator !== this._feedSetIterator) {
-        assert(this._feedSetIterator, 'Iterator not initialized.');
+        invariant(this._feedSetIterator, 'Iterator not initialized.');
         lastFeedSetIterator = this._feedSetIterator;
         iterable = lastFeedSetIterator[Symbol.asyncIterator]();
       }

package/src/space/auth.ts

@@ -2,7 +2,7 @@
 // Copyright 2019 DXOS.org
 //
 
-import assert from 'node:assert';
+import invariant from 'tiny-invariant';
 
 import { runInContext, scheduleTask } from '@dxos/async';
 import { Context } from '@dxos/context';
@@ -67,9 +67,9 @@ export class AuthExtension extends RpcExtension<Services, Services> {
       try {
         const challenge = randomBytes(32);
         const { credential } = await this.rpc.AuthService.authenticate({ challenge });
-        assert(credential?.length > 0, 'invalid credential');
+        invariant(credential?.length > 0, 'invalid credential');
         const success = await this._authParams.verifier(challenge, credential);
-        assert(success, 'credential not verified');
+        invariant(success, 'credential not verified');
         runInContext(this._ctx, () => this._authParams.onAuthSuccess());
       } catch (err) {
         log('auth failed', err);

package/src/space/data-pipeline.ts

@@ -2,7 +2,7 @@
 // Copyright 2022 DXOS.org
 //
 
-import assert from 'node:assert';
+import invariant from 'tiny-invariant';
 
 import { Event, scheduleTask, synchronized, trackLeaks } from '@dxos/async';
 import { Context } from '@dxos/context';
@@ -65,10 +65,10 @@ const TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
  * Reacts to new epochs to restart the pipeline.
  */
 @trackLeaks('open', 'close')
-export class DataPipeline {
+export class DataPipeline implements CredentialProcessor {
   private _ctx = new Context();
-  private _pipeline?: Pipeline;
-  private _targetTimeframe?: Timeframe;
+  private _pipeline?: Pipeline = undefined;
+  private _targetTimeframe?: Timeframe = undefined;
 
   private _lastAutomaticSnapshotTimeframe = new Timeframe();
   private _isOpen = false;
@@ -84,17 +84,18 @@ export class DataPipeline {
   /**
    * Current epoch. Might be still processing.
    */
-  public currentEpoch?: SpecificCredential<Epoch>;
+  public currentEpoch?: SpecificCredential<Epoch> = undefined;
 
   /**
    * Epoch currently applied.
    */
-  public appliedEpoch?: SpecificCredential<Epoch>;
+  public appliedEpoch?: SpecificCredential<Epoch> = undefined;
 
   private _lastProcessedEpoch = -1;
-  public onNewEpoch = new Event<Credential>();
   private _epochCtx?: Context;
 
+  public onNewEpoch = new Event<Credential>();
+
   get isOpen() {
     return this._isOpen;
   }
@@ -112,20 +113,16 @@ export class DataPipeline {
     this._pipeline?.state.setTargetTimeframe(timeframe);
   }
 
-  createCredentialProcessor(): CredentialProcessor {
-    return {
-      process: async (credential) => {
-        if (!checkCredentialType(credential, 'dxos.halo.credentials.Epoch')) {
-          return;
-        }
+  async processCredential(credential: Credential) {
+    if (!checkCredentialType(credential, 'dxos.halo.credentials.Epoch')) {
+      return;
+    }
 
-        this.currentEpoch = credential;
-        if (this._isOpen) {
-          // process epoch
-          await this._processEpochInSeparateTask(credential);
-        }
-      },
-    };
+    this.currentEpoch = credential;
+    if (this._isOpen) {
+      // process epoch
+      await this._processEpochInSeparateTask(credential);
+    }
   }
 
   @synchronized
@@ -146,8 +143,8 @@ export class DataPipeline {
     // Create database backend.
     const feedWriter: FeedWriter<DataMessage> = {
       write: (data, options) => {
-        assert(this._pipeline, 'Pipeline is not initialized.');
-        assert(this.currentEpoch, 'Epoch is not initialized.');
+        invariant(this._pipeline, 'Pipeline is not initialized.');
+        invariant(this.currentEpoch, 'Epoch is not initialized.');
         return this._pipeline.writer.write({ data }, options);
       },
     };
@@ -189,6 +186,15 @@ export class DataPipeline {
 
     await this.databaseHost?.close();
     await this.itemManager?.destroy();
+
+    this._ctx = new Context();
+    this._pipeline = undefined;
+    this._targetTimeframe = undefined;
+    this._lastAutomaticSnapshotTimeframe = new Timeframe();
+    this.currentEpoch = undefined;
+    this.appliedEpoch = undefined;
+    this._lastProcessedEpoch = -1;
+    this._epochCtx = undefined;
   }
 
   private async _consumePipeline() {
@@ -198,7 +204,7 @@ export class DataPipeline {
       await waitForOneEpoch;
     }
 
-    assert(this._pipeline, 'Pipeline is not initialized.');
+    invariant(this._pipeline, 'Pipeline is not initialized.');
     for await (const msg of this._pipeline.consume()) {
       const { feedKey, seq, data } = msg;
       log('processing message', { feedKey, seq });
@@ -237,7 +243,7 @@ export class DataPipeline {
   }
 
   private _createSnapshot(): SpaceSnapshot {
-    assert(this.databaseHost, 'Database backend is not initialized.');
+    invariant(this.databaseHost, 'Database backend is not initialized.');
     return {
       spaceKey: this._params.spaceKey.asUint8Array(),
       timeframe: this._pipeline!.state.timeframe,
@@ -259,6 +265,7 @@ export class DataPipeline {
       const propertiesItem = this.itemManager.items.find(
         (item) =>
           item.modelMeta?.type === 'dxos:model/document' &&
+          // TODO(burdon): Document?
           (getStateMachineFromItem(item)?.snapshot() as ObjectSnapshot).type === 'dxos.sdk.client.Properties',
       );
       if (propertiesItem) {
@@ -319,8 +326,8 @@ export class DataPipeline {
 
   @synchronized
   private async _processEpoch(ctx: Context, epoch: Epoch) {
-    assert(this._isOpen, 'Space is closed.');
-    assert(this._pipeline);
+    invariant(this._isOpen, 'Space is closed.');
+    invariant(this._pipeline);
     this._lastProcessedEpoch = epoch.number;
 
     log('Processing epoch', { epoch });
@@ -339,14 +346,14 @@ export class DataPipeline {
   }
 
   async waitUntilTimeframe(timeframe: Timeframe) {
-    assert(this._pipeline, 'Pipeline is not initialized.');
+    invariant(this._pipeline, 'Pipeline is not initialized.');
     await this._pipeline.state.waitUntilTimeframe(timeframe);
   }
 
   @synchronized
   async createEpoch(): Promise<Epoch> {
-    assert(this._pipeline);
-    assert(this.currentEpoch);
+    invariant(this._pipeline);
+    invariant(this.currentEpoch);
 
     await this._pipeline.pause();
 

package/src/space/space-protocol.browser.test.ts

@@ -43,8 +43,8 @@ describe('space/space-protocol', () => {
     afterTest(() => protocol2.stop());
 
     await waitForExpect(() => {
-      expect(presence1.getPeersOnline().map(({ identityKey }) => identityKey)).toContainEqual(peer2.identityKey);
-      expect(presence2.getPeersOnline().map(({ identityKey }) => identityKey)).toContainEqual(peer1.identityKey);
+      expect(presence1.getPeersOnline().some(({ identityKey }) => identityKey.equals(peer2.identityKey))).toBeTruthy();
+      expect(presence2.getPeersOnline().some(({ identityKey }) => identityKey.equals(peer1.identityKey))).toBeTruthy();
     });
   });
 

package/src/space/space-protocol.test.ts

@@ -39,8 +39,8 @@ describe('space/space-protocol', () => {
     afterTest(() => protocol2.stop());
 
     await waitForExpect(() => {
-      expect(presence1.getPeersOnline().map(({ identityKey }) => identityKey)).toContainEqual(peer2.identityKey);
-      expect(presence2.getPeersOnline().map(({ identityKey }) => identityKey)).toContainEqual(peer1.identityKey);
+      expect(presence1.getPeersOnline().some(({ identityKey }) => identityKey.equals(peer2.identityKey))).toBeTruthy();
+      expect(presence2.getPeersOnline().some(({ identityKey }) => identityKey.equals(peer1.identityKey))).toBeTruthy();
     }, 1_000);
   });
 

package/src/space/space-protocol.ts

@@ -2,7 +2,8 @@
 // Copyright 2022 DXOS.org
 //
 
-import { discoveryKey, sha256 } from '@dxos/crypto';
+import { Event } from '@dxos/async';
+import { discoveryKey, subtleCrypto } from '@dxos/crypto';
 import { FeedWrapper } from '@dxos/feed-store';
 import { PublicKey } from '@dxos/keys';
 import { log, logInfo } from '@dxos/log';
@@ -14,6 +15,7 @@ import {
   WireProtocolParams,
   WireProtocolProvider,
 } from '@dxos/network-manager';
+import { ConnectionInfo } from '@dxos/protocols/proto/dxos/devtools/swarm';
 import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { Teleport } from '@dxos/teleport';
 import { BlobStore, BlobSync } from '@dxos/teleport-extension-object-sync';
@@ -59,7 +61,7 @@ export class SpaceProtocol {
   public readonly blobSync: BlobSync;
 
   @logInfo
-  private readonly _topic: PublicKey;
+  private readonly _topic: Promise<PublicKey>;
 
   private readonly _feeds = new Set<FeedWrapper<FeedMessage>>();
   private readonly _sessions = new ComplexMap<PublicKey, SpaceProtocolSession>(PublicKey.hash);
@@ -86,7 +88,7 @@ export class SpaceProtocol {
     this._onAuthFailure = onAuthFailure;
     this.blobSync = new BlobSync({ blobStore });
 
-    this._topic = PublicKey.from(discoveryKey(sha256(topic.toHex())));
+    this._topic = subtleCrypto.digest('SHA-256', topic.asBuffer()).then(discoveryKey).then(PublicKey.from);
   }
 
   // TODO(burdon): Create abstraction for Space (e.g., add keys and have provider).
@@ -120,7 +122,7 @@ export class SpaceProtocol {
     this._connection = await this._networkManager.joinSwarm({
       protocolProvider: this._createProtocolProvider(credentials),
       peerId: this._swarmIdentity.peerKey,
-      topic: this._topic,
+      topic: await this._topic,
       topology: new MMSTTopology(topologyConfig),
       label: `Protocol swarm: ${this._topic}`,
     });
@@ -204,6 +206,10 @@ export class SpaceProtocolSession implements WireProtocol {
     return this._authStatus;
   }
 
+  get stats(): Event<ConnectionInfo.StreamStats[]> {
+    return this._teleport.stats;
+  }
+
   // TODO(dmaretskyi): Allow to pass in extra extensions.
   constructor({ wireParams, swarmIdentity, onSessionAuth, onAuthFailure, blobSync }: SpaceProtocolSessionParams) {
     this._wireParams = wireParams;

package/src/space/space.test.ts

@@ -22,7 +22,6 @@ describe('space/space', () => {
     const space = await agent.createSpace();
 
     await space.open();
-
     expect(space.isOpen).toBeTruthy();
     afterTest(() => space.close());
 
@@ -130,7 +129,6 @@ describe('space/space', () => {
     const space1 = await agent.createSpace();
 
     await space1.open();
-
     expect(space1.isOpen).toBeTruthy();
     afterTest(() => space1.close());
 
@@ -152,7 +150,6 @@ describe('space/space', () => {
     const space2 = await agent.createSpace(agent.identityKey, space1.key, space1.genesisFeedKey, space1.dataFeedKey);
 
     await space2.open();
-
     await space2.controlPipeline.state!.waitUntilTimeframe(space2.controlPipeline.state!.endTimeframe);
     await space2.initializeDataPipeline();
     await space2.dataPipeline.ensureEpochInitialized();
@@ -166,6 +163,52 @@ describe('space/space', () => {
     await testLocalDatabase(space2.dataPipeline);
   });
 
+  test('re-open', async () => {
+    const builder = new TestAgentBuilder();
+    afterTest(async () => await builder.close());
+    const agent = await builder.createPeer();
+    const space = await agent.createSpace();
+
+    let objectCount: number;
+    {
+      await space.open();
+      afterTest(() => space.close());
+      expect(space.isOpen).toBeTruthy();
+
+      await agent.spaceGenesis(space);
+
+      await space.controlPipeline.state!.waitUntilTimeframe(space.controlPipeline.state!.endTimeframe);
+      await space.initializeDataPipeline();
+      await space.dataPipeline.ensureEpochInitialized();
+
+      assert(space.dataPipeline.databaseHost);
+      await testLocalDatabase(space.dataPipeline);
+
+      objectCount = space.dataPipeline.itemManager.entities.size;
+
+      await space.close();
+      expect(space.isOpen).toBeFalsy();
+    }
+
+    // Re-open.
+    {
+      await space.open();
+      expect(space.isOpen).toBeTruthy();
+
+      await space.controlPipeline.state!.waitUntilTimeframe(space.controlPipeline.state!.endTimeframe);
+      await space.initializeDataPipeline();
+      await space.dataPipeline.ensureEpochInitialized();
+
+      space.dataPipeline.setTargetTimeframe(space.dataPipeline.pipelineState!.endTimeframe);
+      await space.dataPipeline.pipelineState!.waitUntilReachedTargetTimeframe();
+
+      assert(space.dataPipeline.databaseHost);
+      expect(space.dataPipeline.itemManager.entities.size).toEqual(objectCount);
+
+      await testLocalDatabase(space.dataPipeline);
+    }
+  });
+
   test('create epoch', async () => {
     const builder = new TestAgentBuilder();
     afterTest(async () => await builder.close());
@@ -173,7 +216,6 @@ describe('space/space', () => {
     const space1 = await agent.createSpace();
 
     await space1.open();
-
     expect(space1.isOpen).toBeTruthy();
     afterTest(() => space1.close());
 
@@ -214,7 +256,6 @@ describe('space/space', () => {
     const space2 = await agent.createSpace(agent.identityKey, space1.key, space1.genesisFeedKey, space1.dataFeedKey);
 
     await space2.open();
-
     await space2.controlPipeline.state!.waitUntilTimeframe(space2.controlPipeline.state!.endTimeframe);
     await space2.initializeDataPipeline();
     await space2.dataPipeline.ensureEpochInitialized();

package/src/space/space.ts

@@ -2,10 +2,10 @@
 // Copyright 2022 DXOS.org
 //
 
-import assert from 'node:assert';
+import invariant from 'tiny-invariant';
 
 import { Event, synchronized, trackLeaks, Lock } from '@dxos/async';
-import { CredentialConsumer, FeedInfo } from '@dxos/credentials';
+import { FeedInfo } from '@dxos/credentials';
 import { FeedOptions, FeedWrapper } from '@dxos/feed-store';
 import { PublicKey } from '@dxos/keys';
 import { log, logInfo } from '@dxos/log';
@@ -63,15 +63,12 @@ export class Space {
   private readonly _dataPipeline: DataPipeline;
   private readonly _snapshotManager: SnapshotManager;
 
-  // Processes epoch credentials.
-  private _dataPipelineCredentialConsumer?: CredentialConsumer<any> = undefined;
-
   private _isOpen = false;
   private _controlFeed?: FeedWrapper<FeedMessage>;
   private _dataFeed?: FeedWrapper<FeedMessage>;
 
   constructor(params: SpaceParams) {
-    assert(params.spaceKey && params.feedProvider);
+    invariant(params.spaceKey && params.feedProvider);
     this._key = params.spaceKey;
     this._genesisFeedKey = params.genesisFeed.key;
     this._feedProvider = params.feedProvider;
@@ -181,14 +178,14 @@ export class Space {
   }
 
   setControlFeed(feed: FeedWrapper<FeedMessage>) {
-    assert(!this._controlFeed, 'Control feed already set.');
+    invariant(!this._controlFeed, 'Control feed already set.');
     this._controlFeed = feed;
     this._controlPipeline.setWriteFeed(feed);
     return this;
   }
 
   setDataFeed(feed: FeedWrapper<FeedMessage>) {
-    assert(!this._dataFeed, 'Data feed already set.');
+    invariant(!this._dataFeed, 'Data feed already set.');
     this._dataFeed = feed;
     this._dataPipeline.pipeline?.setWriteFeed(feed);
     return this;
@@ -217,12 +214,7 @@ export class Space {
     // Order is important.
     await this._controlPipeline.start();
     await this.protocol.start();
-
-    this._dataPipelineCredentialConsumer = this._controlPipeline.spaceState.registerProcessor(
-      this._dataPipeline.createCredentialProcessor(),
-    );
-
-    await this._dataPipelineCredentialConsumer.open();
+    await this._controlPipeline.spaceState.addCredentialProcessor(this._dataPipeline);
 
     this._isOpen = true;
     log('opened');
@@ -234,8 +226,7 @@ export class Space {
     if (!this._isOpen) {
       return;
     }
-    await this._dataPipelineCredentialConsumer?.close();
-    this._dataPipelineCredentialConsumer = undefined;
+    await this._controlPipeline.spaceState.removeCredentialProcessor(this._dataPipeline);
 
     await this._dataPipeline.close();
 
@@ -250,7 +241,7 @@ export class Space {
   @synchronized
   async initializeDataPipeline() {
     log('initializeDataPipeline');
-    assert(this._isOpen, 'Space must be open to initialize data pipeline.');
+    invariant(this._isOpen, 'Space must be open to initialize data pipeline.');
     await this._dataPipeline.open();
   }
 }

package/src/testing/util.ts

@@ -45,7 +45,7 @@ export const createRemoteDatabaseFromDataServiceHost = async (
   const dataService = new DataServiceImpl(dataServiceSubscriptions);
 
   const spaceKey = PublicKey.random();
-  dataServiceSubscriptions.registerSpace(spaceKey, dataServiceHost);
+  await dataServiceSubscriptions.registerSpace(spaceKey, dataServiceHost);
 
   const itemManager = new ItemManager(modelFactory);
   const backend = new DatabaseProxy(dataService, itemManager, spaceKey);

package/src/tests/database.test.ts

@@ -42,7 +42,7 @@ const createDatabaseWithFeeds = async () => {
   const dataService = new DataServiceImpl(dataServiceSubscriptions);
 
   const spaceKey = PublicKey.random();
-  dataServiceSubscriptions.registerSpace(spaceKey, host.createDataServiceHost());
+  await dataServiceSubscriptions.registerSpace(spaceKey, host.createDataServiceHost());
 
   const proxy = new DatabaseProxy(dataService, new ItemManager(modelFactory), spaceKey);
   await proxy.open(new ModelFactory().registerModel(DocumentModel));