@dxos/echo-pipeline 0.1.56-main.e47dfd1 → 0.1.56-main.e79d64a

package/src/space/control-pipeline.ts

@@ -2,13 +2,16 @@
 // Copyright 2022 DXOS.org
 //
 
+import { Context } from '@dxos/context';
 import { SpaceStateMachine, SpaceState, MemberInfo, FeedInfo } from '@dxos/credentials';
 import { FeedWrapper } from '@dxos/feed-store';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
+import { FeedMessageBlock } from '@dxos/protocols';
 import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { AdmittedFeed, Credential } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { Timeframe } from '@dxos/timeframe';
+import { TimeSeriesCounter, TimeUsageCounter, trace } from '@dxos/tracing';
 import { AsyncCallback, Callback, tracer } from '@dxos/util';
 
 import { MetadataStore } from '../metadata';
@@ -26,6 +29,7 @@ const TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
 /**
  * Processes HALO credentials, which include genesis and invitations.
  */
+@trace.resource()
 export class ControlPipeline {
   private readonly _pipeline: Pipeline;
   private readonly _spaceStateMachine: SpaceStateMachine;
@@ -39,6 +43,12 @@ export class ControlPipeline {
   public readonly onMemberAdmitted: Callback<AsyncCallback<MemberInfo>>;
   public readonly onCredentialProcessed: Callback<AsyncCallback<Credential>>;
 
+  @trace.metricsCounter()
+  private _usage = new TimeUsageCounter();
+
+  @trace.metricsCounter()
+  private _mutations = new TimeSeriesCounter();
+
   constructor({ spaceKey, genesisFeed, feedProvider, metadataStore }: ControlPipelineParams) {
     this._spaceKey = spaceKey;
     this._metadata = metadataStore;
@@ -82,34 +92,49 @@ export class ControlPipeline {
   async start() {
     log('starting...');
     setTimeout(async () => {
-      for await (const msg of this._pipeline.consume()) {
-        try {
-          // log('processing', { msg });
-          log('processing', { key: msg.feedKey, seq: msg.seq });
-          if (msg.data.payload.credential) {
-            const timer = tracer.mark('dxos.echo.pipeline.control');
-            const result = await this._spaceStateMachine.process(
-              msg.data.payload.credential.credential,
-              PublicKey.from(msg.feedKey),
-            );
-
-            timer.end();
-            if (!result) {
-              log.warn('processing failed', { msg });
-            } else {
-              await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
-            }
-          }
-        } catch (err: any) {
-          log.catch(err);
-        }
-      }
+      void this._consumePipeline(new Context());
     });
 
     await this._pipeline.start();
     log('started');
   }
 
+  @trace.span()
+  private async _consumePipeline(ctx: Context) {
+    for await (const msg of this._pipeline.consume()) {
+      const span = this._usage.beginRecording();
+      this._mutations.inc();
+
+      try {
+        await this._processMessage(ctx, msg);
+      } catch (err: any) {
+        log.catch(err);
+      }
+
+      span.end();
+    }
+  }
+
+  @trace.span()
+  private async _processMessage(ctx: Context, msg: FeedMessageBlock) {
+    // log('processing', { msg });
+    log('processing', { key: msg.feedKey, seq: msg.seq });
+    if (msg.data.payload.credential) {
+      const timer = tracer.mark('dxos.echo.pipeline.control');
+      const result = await this._spaceStateMachine.process(
+        msg.data.payload.credential.credential,
+        PublicKey.from(msg.feedKey),
+      );
+
+      timer.end();
+      if (!result) {
+        log.warn('processing failed', { msg });
+      } else {
+        await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
+      }
+    }
+  }
+
   private async _noteTargetStateIfNeeded(timeframe: Timeframe) {
     // TODO(dmaretskyi): Replace this with a proper debounce/throttle.
 

package/src/space/data-pipeline.ts

@@ -2,16 +2,15 @@
 // Copyright 2022 DXOS.org
 //
 
-import invariant from 'tiny-invariant';
-
 import { Event, scheduleTask, synchronized, trackLeaks } from '@dxos/async';
 import { Context } from '@dxos/context';
 import { CredentialProcessor, FeedInfo, SpecificCredential, checkCredentialType } from '@dxos/credentials';
-import { getStateMachineFromItem, ItemManager } from '@dxos/echo-db';
+import { getStateMachineFromItem, ItemManager, TYPE_PROPERTIES } from '@dxos/echo-db';
 import { CancelledError } from '@dxos/errors';
 import { FeedWriter } from '@dxos/feed-store';
+import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
-import { log } from '@dxos/log';
+import { log, omit } from '@dxos/log';
 import { ModelFactory } from '@dxos/model-factory';
 import { DataPipelineProcessed } from '@dxos/protocols';
 import { DataMessage } from '@dxos/protocols/proto/dxos/echo/feed';
@@ -20,6 +19,7 @@ import { ObjectSnapshot } from '@dxos/protocols/proto/dxos/echo/model/document';
 import { SpaceSnapshot } from '@dxos/protocols/proto/dxos/echo/snapshot';
 import { Credential, Epoch } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { Timeframe } from '@dxos/timeframe';
+import { TimeSeriesCounter, TimeUsageCounter, trace } from '@dxos/tracing';
 import { tracer } from '@dxos/util';
 
 import { DatabaseHost, SnapshotManager } from '../db-host';
@@ -66,6 +66,7 @@ const TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
  * Reacts to new epochs to restart the pipeline.
  */
 @trackLeaks('open', 'close')
+@trace.resource()
 export class DataPipeline implements CredentialProcessor {
   private _ctx = new Context();
   private _pipeline?: Pipeline = undefined;
@@ -79,6 +80,12 @@ export class DataPipeline implements CredentialProcessor {
   private _lastProcessedEpoch = -1;
   private _epochCtx?: Context;
 
+  @trace.metricsCounter()
+  private _usage = new TimeUsageCounter();
+
+  @trace.metricsCounter()
+  private _mutations = new TimeSeriesCounter();
+
   constructor(private readonly _params: DataPipelineParams) {}
 
   public itemManager!: ItemManager;
@@ -133,8 +140,9 @@ export class DataPipeline implements CredentialProcessor {
 
     this._pipeline = new Pipeline();
     await this._params.onPipelineCreated(this._pipeline);
-    await this._pipeline.start();
+
     await this._pipeline.pause(); // Start paused until we have the first epoch.
+    await this._pipeline.start();
 
     if (this._targetTimeframe) {
       this._pipeline.state.setTargetTimeframe(this._targetTimeframe);
@@ -206,6 +214,9 @@ export class DataPipeline implements CredentialProcessor {
 
     invariant(this._pipeline, 'Pipeline is not initialized.');
     for await (const msg of this._pipeline.consume()) {
+      const span = this._usage.beginRecording();
+      this._mutations.inc();
+
       const { feedKey, seq, data } = msg;
       log('processing message', { feedKey, seq });
 
@@ -242,6 +253,8 @@ export class DataPipeline implements CredentialProcessor {
       } catch (err: any) {
         log.catch(err);
       }
+
+      span.end();
     }
   }
 
@@ -269,7 +282,7 @@ export class DataPipeline implements CredentialProcessor {
         (item) =>
           item.modelMeta?.type === 'dxos:model/document' &&
           // TODO(burdon): Document?
-          (getStateMachineFromItem(item)?.snapshot() as ObjectSnapshot).type === 'dxos.sdk.client.Properties',
+          (getStateMachineFromItem(item)?.snapshot() as ObjectSnapshot).type === TYPE_PROPERTIES,
       );
       if (propertiesItem) {
         cache.properties = getStateMachineFromItem(propertiesItem)?.snapshot() as ObjectSnapshot;
@@ -319,7 +332,6 @@ export class DataPipeline implements CredentialProcessor {
         // Space closed before we got to process the epoch.
         return;
       }
-      log('process epoch', { epoch });
       await this._processEpoch(ctx, epoch.subject.assertion);
 
       this.appliedEpoch = epoch;
@@ -333,16 +345,13 @@ export class DataPipeline implements CredentialProcessor {
     invariant(this._pipeline);
     this._lastProcessedEpoch = epoch.number;
 
-    log('Processing epoch', { epoch });
+    log('processing', { epoch: omit(epoch, 'proof') });
     if (epoch.snapshotCid) {
       const snapshot = await this._params.snapshotManager.load(ctx, epoch.snapshotCid);
-
-      // TODO(dmaretskyi): Clearing old items + events.
       this.databaseHost!._itemDemuxer.restoreFromSnapshot(snapshot.database);
     }
 
-    log('restarting pipeline for epoch');
-
+    log('restarting pipeline from epoch');
     await this._pipeline.pause();
     await this._pipeline.setCursor(epoch.timeframe);
     await this._pipeline.unpause();

package/src/space/space-manager.ts

@@ -122,6 +122,7 @@ export class SpaceManager {
       memberKey,
     });
     this._spaces.set(space.key, space);
+
     log.trace('dxos.echo.space-manager.construct-space', trace.end({ id: this._instanceId }));
     return space;
   }

package/src/space/space-protocol.ts

@@ -87,6 +87,7 @@ export class SpaceProtocol {
     this._onAuthFailure = onAuthFailure;
     this.blobSync = new BlobSync({ blobStore });
 
+    // TODO(burdon): Async race condition? Move to start?
     this._topic = subtleCrypto.digest('SHA-256', topic.asBuffer()).then(discoveryKey).then(PublicKey.from);
   }
 
@@ -100,6 +101,7 @@ export class SpaceProtocol {
     }
   }
 
+  // TODO(burdon): Rename open? Common open/close interfaces for all services?
   async start() {
     if (this._connection) {
       return;
@@ -124,7 +126,7 @@ export class SpaceProtocol {
       peerId: this._swarmIdentity.peerKey,
       topic,
       topology: new MMSTTopology(topologyConfig),
-      label: `Protocol swarm: ${topic}`,
+      label: `space swarm ${topic.truncate()}`,
     });
 
     log('started');

package/src/space/space.test.ts

@@ -3,10 +3,11 @@
 //
 
 import expect from 'expect';
-import assert from 'node:assert';
 import { promisify } from 'node:util';
 
+import { Context } from '@dxos/context';
 import { createCredential, CredentialGenerator } from '@dxos/credentials';
+import { invariant } from '@dxos/invariant';
 import { afterTest, describe, test } from '@dxos/test';
 
 import { TestAgentBuilder, testLocalDatabase } from '../testing';
@@ -21,7 +22,7 @@ describe('space/space', () => {
     const agent = await builder.createPeer();
     const space = await agent.createSpace();
 
-    await space.open();
+    await space.open(new Context());
     expect(space.isOpen).toBeTruthy();
     afterTest(() => space.close());
 
@@ -31,7 +32,7 @@ describe('space/space', () => {
     await space.initializeDataPipeline();
     await space.dataPipeline.ensureEpochInitialized();
 
-    assert(space.dataPipeline.databaseHost);
+    invariant(space.dataPipeline.databaseHost);
     await testLocalDatabase(space.dataPipeline);
 
     await builder.close();
@@ -49,7 +50,7 @@ describe('space/space', () => {
       const agent = await builder.createPeer();
       const space = await agent.createSpace(agent.identityKey);
 
-      await space.open();
+      await space.open(new Context());
       expect(space.isOpen).toBeTruthy();
       afterTest(() => space.close());
 
@@ -70,7 +71,7 @@ describe('space/space', () => {
       const agent = await builder.createPeer();
       const space = await agent.createSpace(agent.identityKey, space1.key, space1.genesisFeedKey, undefined, true);
 
-      await space.open();
+      await space.open(new Context());
       expect(space.isOpen).toBeTruthy();
       afterTest(() => space.close());
 
@@ -128,7 +129,7 @@ describe('space/space', () => {
     const agent = await builder.createPeer();
     const space1 = await agent.createSpace();
 
-    await space1.open();
+    await space1.open(new Context());
     expect(space1.isOpen).toBeTruthy();
     afterTest(() => space1.close());
 
@@ -138,7 +139,7 @@ describe('space/space', () => {
     await space1.initializeDataPipeline();
     await space1.dataPipeline.ensureEpochInitialized();
 
-    assert(space1.dataPipeline.databaseHost);
+    invariant(space1.dataPipeline.databaseHost);
     await testLocalDatabase(space1.dataPipeline);
 
     const objectCount = space1.dataPipeline.itemManager.entities.size;
@@ -149,7 +150,7 @@ describe('space/space', () => {
     // Re-open.
     const space2 = await agent.createSpace(agent.identityKey, space1.key, space1.genesisFeedKey, space1.dataFeedKey);
 
-    await space2.open();
+    await space2.open(new Context());
     await space2.controlPipeline.state!.waitUntilTimeframe(space2.controlPipeline.state!.endTimeframe);
     await space2.initializeDataPipeline();
     await space2.dataPipeline.ensureEpochInitialized();
@@ -157,7 +158,7 @@ describe('space/space', () => {
     space2.dataPipeline.setTargetTimeframe(space2.dataPipeline.pipelineState!.endTimeframe);
     await space2.dataPipeline.pipelineState!.waitUntilReachedTargetTimeframe();
 
-    assert(space2.dataPipeline.databaseHost);
+    invariant(space2.dataPipeline.databaseHost);
     expect(space2.dataPipeline.itemManager.entities.size).toEqual(objectCount);
 
     await testLocalDatabase(space2.dataPipeline);
@@ -171,7 +172,7 @@ describe('space/space', () => {
 
     let objectCount: number;
     {
-      await space.open();
+      await space.open(new Context());
       afterTest(() => space.close());
       expect(space.isOpen).toBeTruthy();
 
@@ -181,7 +182,7 @@ describe('space/space', () => {
       await space.initializeDataPipeline();
       await space.dataPipeline.ensureEpochInitialized();
 
-      assert(space.dataPipeline.databaseHost);
+      invariant(space.dataPipeline.databaseHost);
       await testLocalDatabase(space.dataPipeline);
 
       objectCount = space.dataPipeline.itemManager.entities.size;
@@ -192,7 +193,7 @@ describe('space/space', () => {
 
     // Re-open.
     {
-      await space.open();
+      await space.open(new Context());
       expect(space.isOpen).toBeTruthy();
 
       await space.controlPipeline.state!.waitUntilTimeframe(space.controlPipeline.state!.endTimeframe);
@@ -202,7 +203,7 @@ describe('space/space', () => {
       space.dataPipeline.setTargetTimeframe(space.dataPipeline.pipelineState!.endTimeframe);
       await space.dataPipeline.pipelineState!.waitUntilReachedTargetTimeframe();
 
-      assert(space.dataPipeline.databaseHost);
+      invariant(space.dataPipeline.databaseHost);
       expect(space.dataPipeline.itemManager.entities.size).toEqual(objectCount);
 
       await testLocalDatabase(space.dataPipeline);
@@ -215,7 +216,7 @@ describe('space/space', () => {
     const agent = await builder.createPeer();
     const space1 = await agent.createSpace();
 
-    await space1.open();
+    await space1.open(new Context());
     expect(space1.isOpen).toBeTruthy();
     afterTest(() => space1.close());
 
@@ -225,7 +226,7 @@ describe('space/space', () => {
     await space1.initializeDataPipeline();
     await space1.dataPipeline.ensureEpochInitialized();
 
-    assert(space1.dataPipeline.databaseHost);
+    invariant(space1.dataPipeline.databaseHost);
     await testLocalDatabase(space1.dataPipeline);
 
     const objectCount = space1.dataPipeline.itemManager.entities.size;
@@ -255,7 +256,7 @@ describe('space/space', () => {
     // Re-open.
     const space2 = await agent.createSpace(agent.identityKey, space1.key, space1.genesisFeedKey, space1.dataFeedKey);
 
-    await space2.open();
+    await space2.open(new Context());
     await space2.controlPipeline.state!.waitUntilTimeframe(space2.controlPipeline.state!.endTimeframe);
     await space2.initializeDataPipeline();
     await space2.dataPipeline.ensureEpochInitialized();
@@ -263,7 +264,7 @@ describe('space/space', () => {
     space2.dataPipeline.setTargetTimeframe(space2.dataPipeline.pipelineState!.endTimeframe);
     await space2.dataPipeline.pipelineState!.waitUntilReachedTargetTimeframe();
 
-    assert(space2.dataPipeline.databaseHost);
+    invariant(space2.dataPipeline.databaseHost);
     expect(space2.dataPipeline.itemManager.entities.size).toEqual(objectCount);
 
     await testLocalDatabase(space2.dataPipeline);

package/src/space/space.ts

@@ -2,17 +2,18 @@
 // Copyright 2022 DXOS.org
 //
 
-import invariant from 'tiny-invariant';
-
 import { Event, synchronized, trackLeaks, Lock } from '@dxos/async';
+import { Context } from '@dxos/context';
 import { FeedInfo } from '@dxos/credentials';
 import { FeedOptions, FeedWrapper } from '@dxos/feed-store';
+import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log, logInfo } from '@dxos/log';
 import { ModelFactory } from '@dxos/model-factory';
 import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { AdmittedFeed, Credential } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { Timeframe } from '@dxos/timeframe';
+import { trace } from '@dxos/tracing';
 import { AsyncCallback, Callback } from '@dxos/util';
 
 import { SnapshotManager } from '../db-host';
@@ -49,6 +50,7 @@ export type CreatePipelineParams = {
  */
 // TODO(dmaretskyi): Extract database stuff.
 @trackLeaks('open', 'close')
+@trace.resource()
 export class Space {
   private readonly _addFeedLock = new Lock();
 
@@ -138,6 +140,7 @@ export class Space {
   }
 
   @logInfo
+  @trace.info()
   get key() {
     return this._key;
   }
@@ -205,7 +208,8 @@ export class Space {
   //   return this._dataPipeline?.getFeeds();
   // }
   @synchronized
-  async open() {
+  @trace.span()
+  async open(ctx: Context) {
     log('opening...');
     if (this._isOpen) {
       return;

package/src/testing/database-test-rig.ts

@@ -8,9 +8,10 @@ import { DatabaseProxy, ItemManager } from '@dxos/echo-db';
 import { WriteOptions, WriteReceipt } from '@dxos/feed-store';
 import { PublicKey } from '@dxos/keys';
 import { ModelFactory } from '@dxos/model-factory';
-import { FeedMessageBlock } from '@dxos/protocols';
+import { FeedMessageBlock, schema } from '@dxos/protocols';
 import { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
-import { SpaceSnapshot } from '@dxos/protocols/proto/dxos/echo/snapshot';
+import { EchoSnapshot, SpaceSnapshot } from '@dxos/protocols/proto/dxos/echo/snapshot';
+import { Epoch } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { TextModel } from '@dxos/text-model';
 import { Timeframe } from '@dxos/timeframe';
 import { ComplexMap, isNotNullOrUndefined } from '@dxos/util';
@@ -53,6 +54,9 @@ export class DatabaseTestPeer {
 
   public feedMessages: FeedMessage[] = [];
 
+  public readonly snapshots = new Map<string, SpaceSnapshot>();
+  private currentEpoch?: Epoch;
+
   /**
    * Sequence number of the last mutation confirmed to be written to the feed store.
    */
@@ -162,6 +166,27 @@ export class DatabaseTestPeer {
     return this.snapshot;
   }
 
+  createEpoch(mockSnapshot?: EchoSnapshot) {
+    const snapshot = this.makeSnapshot();
+    // Substitute snapshot with the mock one for test purposes (e.g. to test with empty snapshot).
+    mockSnapshot && (snapshot.database = mockSnapshot);
+    const snapshotCid = PublicKey.from(
+      schema.getCodecForType('dxos.echo.snapshot.SpaceSnapshot').encode(snapshot),
+    ).toHex();
+    this.snapshots.set(snapshotCid, snapshot);
+
+    const epoch: Epoch = {
+      previousId: PublicKey.random(),
+      timeframe: this.timeframe,
+      number: this.currentEpoch ? this.currentEpoch.number + 1 : 0,
+      snapshotCid,
+    };
+
+    this.currentEpoch = epoch;
+
+    this.host._itemDemuxer.restoreFromSnapshot(snapshot.database);
+  }
+
   /**
    * Gets all candidate messages according to the current timeframe.
    * Does not take into account the current snapshot, timeframe dependencies, or the confirmed, or replicated state.

package/src/testing/test-agent-builder.ts

@@ -2,6 +2,7 @@
 // Copyright 2022 DXOS.org
 //
 
+import { Context } from '@dxos/context';
 import { CredentialGenerator } from '@dxos/credentials';
 import { DocumentModel } from '@dxos/document-model';
 import { FeedStore } from '@dxos/feed-store';
@@ -198,7 +199,7 @@ export class TestAgent {
     space.setControlFeed(controlFeed);
     space.setDataFeed(dataFeed);
 
-    await space.open();
+    await space.open(new Context());
 
     this._spaces.set(spaceKey, space);
     return space;

package/src/tests/database-unit.test.ts

@@ -158,4 +158,43 @@ describe('database (unit)', () => {
       expect(model.get('tags').toArray()).toHaveLength(1);
     });
   });
+
+  it('epoch correctly resets items', async () => {
+    const builder = new DatabaseTestBuilder();
+    const peer = await builder.createPeer();
+
+    const id = PublicKey.random().toHex();
+    peer.proxy.mutate(genesisMutation(id, DocumentModel.meta.type));
+    peer.proxy.mutate(
+      createModelMutation(id, encodeModelMutation(DocumentModel.meta, new MutationBuilder().set('test', 42).build())),
+    );
+
+    await peer.confirm();
+    expect(peer.confirmed).toEqual(1);
+    expect(peer.timeframe).toEqual(new Timeframe([[peer.key, 1]]));
+
+    const state = peer.items.getItem(id)!.state;
+    const snapshotWithItem = peer.makeSnapshot().database;
+
+    {
+      // Reset on empty epoch.
+      const updated = peer.proxy.itemUpdate.waitForCount(1);
+
+      expect(peer.proxy._itemManager.items.length).toEqual(1);
+
+      peer.createEpoch({});
+
+      await updated;
+      expect(peer.proxy._itemManager.items.length).toEqual(0);
+    }
+
+    {
+      // Reset to epoch with one item.
+      const updated = peer.proxy.itemUpdate.waitForCount(1);
+      peer.createEpoch(snapshotWithItem);
+      await updated;
+      expect(peer.proxy._itemManager.items.length).toEqual(1);
+      expect(peer.items.getItem(id)!.state).toEqual(state);
+    }
+  });
 });