@dxos/echo-pipeline 0.1.56-main.c19b7cd → 0.1.56-main.c277f68

package/src/pipeline/pipeline.test.ts

@@ -3,14 +3,8 @@
 //
 
 import expect from 'expect';
-import * as fc from 'fast-check';
-import { inspect } from 'util';
 
-import { Event, asyncTimeout, sleep } from '@dxos/async';
-import { FeedStore, FeedWrapper } from '@dxos/feed-store';
-import { PublicKey } from '@dxos/keys';
-import { log } from '@dxos/log';
-import { FeedMessageBlock } from '@dxos/protocols';
+import { Event, sleep } from '@dxos/async';
 import { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { describe, test, afterTest } from '@dxos/test';
 import { Timeframe } from '@dxos/timeframe';
@@ -85,7 +79,7 @@ describe('pipeline/Pipeline', () => {
 
     const processedSequenceNumbers: number[] = [];
 
-    // skip first 10, process the rest, and the repeat the last 10.
+    // Skip first 10, process the rest, and the repeat the last 10.
     const expectedSequenceNumbers = [...sequenceNumbers.slice(10, 30), ...sequenceNumbers.slice(20, 30)];
 
     await pipeline.setCursor(new Timeframe([[feed.key, 9]]));
@@ -144,210 +138,4 @@ describe('pipeline/Pipeline', () => {
     await processedEvent.waitForCondition(() => processedSequenceNumbers.length === 10);
     expect(processedSequenceNumbers).toEqual(expectedSequenceNumbers);
   });
-
-  test
-    .skip('stress', async () => {
-      const builder = new TestFeedBuilder();
-
-      const NUM_AGENTS = 2;
-      const NUM_MESSAGES = 100;
-
-      class Agent {
-        public startingTimeframe = new Timeframe();
-        public pipeline!: Pipeline;
-        public feed!: FeedWrapper<FeedMessage>;
-        public messages: FeedMessageBlock[] = [];
-        public writePromise: Promise<any> = Promise.resolve();
-
-        constructor(public id: string, public feedStore: FeedStore<FeedMessage>) {}
-
-        async open() {
-          const key = await builder.keyring.createKey();
-          this.feed = await this.feedStore.openFeed(key, { writable: true });
-        }
-
-        async start() {
-          this.pipeline = new Pipeline();
-          await this.pipeline.setCursor(this.startingTimeframe);
-
-          await this.pipeline.start();
-
-          // NOTE: not awaiting here breaks the test.
-          await Promise.all(this.feedStore.feeds.map((feed) => this.pipeline.addFeed(feed)));
-          this.pipeline.setWriteFeed(this.feed);
-
-          // consume in async task.
-          void this.consume();
-        }
-
-        async stop() {
-          await this.pipeline.stop();
-          this.startingTimeframe = this.pipeline.state.timeframe;
-        }
-
-        async close() {
-          await this.feed.close();
-        }
-
-        write(message: FeedMessage.Payload) {
-          const prev = this.writePromise;
-          const promise = this.pipeline.writer!.write(message);
-          this.writePromise = Promise.all([prev, promise]);
-        }
-
-        async consume() {
-          for await (const msg of this.pipeline.consume()) {
-            this.messages.push(msg);
-          }
-          log('stopped consuming');
-        }
-      }
-
-      type Model = {};
-      type Real = {
-        feedStore: FeedStore<FeedMessage>;
-        agents: Map<string, Agent>;
-      };
-
-      class WriteCommand implements fc.AsyncCommand<Model, Real> {
-        constructor(public agent: string, public count: number) {}
-
-        check = () => true;
-
-        async run(model: Model, real: Real) {
-          console.log(`WriteCommand(${this.agent}, ${this.count})`);
-
-          const agent = real.agents.get(this.agent)!;
-
-          const toWrite = Math.min(this.count, NUM_MESSAGES - agent.feed.length);
-          if (toWrite > 0) {
-            for (const _ of range(toWrite)) {
-              agent.write({}); // Content is not important.
-            }
-          }
-        }
-
-        toString = () => `WriteCommand(${this.agent}, ${this.count})`;
-      }
-
-      class SyncCommand implements fc.AsyncCommand<Model, Real> {
-        check = () => true;
-
-        async run(model: Model, real: Real) {
-          console.log('SyncCommand()');
-          const targets: any = {};
-
-          try {
-            for (const agent of real.agents.values()) {
-              await agent.writePromise;
-            }
-
-            for (const agent of real.agents.values()) {
-              targets[agent.id] = agent.pipeline.state.endTimeframe;
-              if (agent.pipeline.state.endTimeframe.isEmpty()) {
-                console.log('empty endtimeframe', {
-                  id: agent.id,
-                  endTimeframe: agent.pipeline.state.endTimeframe,
-                  feeds: agent.pipeline.getFeeds().map((feed) => [feed.key.toString(), feed.length]),
-                });
-              }
-              await asyncTimeout(agent.pipeline.state.waitUntilTimeframe(agent.pipeline.state.endTimeframe), 1000);
-            }
-
-            const tf: Timeframe = real.agents.values().next().value!.pipeline.state.timeframe;
-            for (const agent of real.agents.values()) {
-              expect(agent.pipeline.state.timeframe.equals(tf)).toEqual(true);
-            }
-
-            const totalMessages = real.feedStore.feeds.reduce((acc, feed) => acc + feed.length, 0);
-            for (const agent of real.agents.values()) {
-              expect(agent.messages.length).toEqual(totalMessages);
-            }
-          } catch (err) {
-            console.log(
-              inspect(
-                {
-                  agents: Array.from(real.agents.values()).map((agent) => ({
-                    id: agent.id,
-                    messages: agent.messages.length,
-                    feeds: agent.pipeline.getFeeds().map((feed) => [feed.key, feed.length]),
-                    timeframe: agent.pipeline.state.timeframe,
-                    endTimeframe: agent.pipeline.state.endTimeframe,
-                  })),
-                  feeds: real.feedStore.feeds.map((feed) => [feed.key, feed.length]),
-                  targets,
-                },
-                false,
-                null,
-                true,
-              ),
-            );
-            throw err;
-          }
-        }
-
-        toString = () => 'SyncCommand()';
-      }
-
-      class RestartCommand implements fc.AsyncCommand<Model, Real> {
-        constructor(public agent: string) {}
-
-        check = () => true;
-
-        async run(model: Model, real: Real) {
-          console.log(`RestartCommand(${this.agent})`);
-
-          const agent = real.agents.get(this.agent)!;
-
-          await agent.writePromise;
-          await agent.stop();
-          await agent.start();
-        }
-
-        toString = () => `RestartCommand(${this.agent})`;
-      }
-
-      const agentIds = range(NUM_AGENTS).map(() => PublicKey.random().toHex().slice(0, 8));
-      const anAgentId = fc.constantFrom(...agentIds);
-
-      const commands = fc.commands(
-        [
-          fc.tuple(anAgentId, fc.integer({ min: 1, max: 10 })).map(([agent, count]) => new WriteCommand(agent, count)),
-          fc.constant(new SyncCommand()),
-          anAgentId.map((agent) => new RestartCommand(agent)),
-        ],
-        { size: 'large' },
-      );
-
-      const model = fc.asyncProperty(commands, async (commands) => {
-        const feedStore = builder.createFeedStore();
-
-        const agents = new Map(agentIds.map((id) => [id, new Agent(id, feedStore)]));
-        await Promise.all(Array.from(agents.values()).map((agent) => agent.open()));
-        await Promise.all(Array.from(agents.values()).map((agent) => agent.start()));
-
-        const setup: fc.ModelRunSetup<Model, Real> = () => ({
-          model: {},
-          real: {
-            feedStore,
-            agents,
-          },
-        });
-
-        try {
-          await fc.asyncModelRun(setup, [...commands, new SyncCommand()]);
-        } finally {
-          await Promise.all(Array.from(agents.values()).map((agent) => agent.stop()));
-          await Promise.all(Array.from(agents.values()).map((agent) => agent.close()));
-        }
-      });
-
-      const examples: [commands: Iterable<fc.AsyncCommand<Model, Real, boolean>>][] = [
-        [[new WriteCommand(agentIds[0], 10), new WriteCommand(agentIds[1], 10), new SyncCommand()]],
-        [[new WriteCommand(agentIds[0], 4), new RestartCommand(agentIds[0]), new SyncCommand()]],
-      ];
-
-      await fc.assert(model, { examples });
-    })
-    .timeout(60_000);
 });

package/src/pipeline/pipeline.ts

@@ -204,9 +204,6 @@ export class Pipeline implements PipelineAccessor {
   private readonly _timeframeClock = new TimeframeClock(new Timeframe());
   private readonly _feeds = new ComplexMap<PublicKey, FeedWrapper<FeedMessage>>(PublicKey.hash);
 
-  // Inbound feed stream.
-  private _feedSetIterator?: FeedSetIterator<FeedMessage>;
-
   // External state accessor.
   private readonly _state: PipelineState = new PipelineState(this._feeds, this._timeframeClock);
 
@@ -214,14 +211,20 @@ export class Pipeline implements PipelineAccessor {
   private readonly _processingTrigger = new Trigger().wake();
   private readonly _pauseTrigger = new Trigger().wake();
 
-  private _isStopping = false;
-  private _isStarted = false;
-  private _isOpen = false;
-  private _isPaused = false;
+  // Pending downloads.
+  private readonly _downloads = new ComplexMap<FeedWrapper<FeedMessage>, any>((value) => PublicKey.hash(value.key));
+
+  // Inbound feed stream.
+  private _feedSetIterator?: FeedSetIterator<FeedMessage>;
 
   // Outbound feed writer.
   private _writer: FeedWriter<FeedMessage.Payload> | undefined;
 
+  private _isStopping = false;
+  private _isStarted = false;
+  private _isBeingConsumed = false;
+  private _isPaused = false;
+
   get state() {
     return this._state;
   }
@@ -243,10 +246,14 @@ export class Pipeline implements PipelineAccessor {
   // which might be opening feeds during the mutation processing, which w
   async addFeed(feed: FeedWrapper<FeedMessage>) {
     this._feeds.set(feed.key, feed);
+
     if (this._feedSetIterator) {
       await this._feedSetIterator.addFeed(feed);
     }
-    this._setFeedDownloadState(feed);
+
+    if (this._isStarted && !this._isPaused) {
+      this._setFeedDownloadState(feed);
+    }
   }
 
   setWriteFeed(feed: FeedWrapper<FeedMessage>) {
@@ -270,12 +277,22 @@ export class Pipeline implements PipelineAccessor {
     await this._feedSetIterator!.open();
     this._isStarted = true;
     log('started');
+
+    if (!this._isPaused) {
+      for (const feed of this._feeds.values()) {
+        this._setFeedDownloadState(feed);
+      }
+    }
   }
 
   @synchronized
   async stop() {
     log('stopping...');
     this._isStopping = true;
+    for (const [feed, handle] of this._downloads.entries()) {
+      feed.undownload(handle);
+    }
+    this._downloads.clear();
     await this._feedSetIterator?.close();
     await this._processingTrigger.wait(); // Wait for the in-flight message to be processed.
     await this._state._ctx.dispose();
@@ -297,10 +314,6 @@ export class Pipeline implements PipelineAccessor {
     this._timeframeClock.setTimeframe(timeframe);
 
     // Cancel downloads of mutations before the cursor.
-    for (const feed of this._feeds.values()) {
-      this._setFeedDownloadState(feed);
-    }
-
     if (this._feedSetIterator) {
       await this._feedSetIterator.close();
       await this._initIterator();
@@ -313,7 +326,6 @@ export class Pipeline implements PipelineAccessor {
    */
   @synchronized
   async pause() {
-    invariant(this._isStarted, 'Pipeline is not open.');
     if (this._isPaused) {
       return;
     }
@@ -325,11 +337,14 @@ export class Pipeline implements PipelineAccessor {
 
   @synchronized
   async unpause() {
-    invariant(this._isStarted, 'Pipeline is not open.');
     invariant(this._isPaused, 'Pipeline is not paused.');
 
     this._pauseTrigger.wake();
     this._isPaused = false;
+
+    for (const feed of this._feeds.values()) {
+      this._setFeedDownloadState(feed);
+    }
   }
 
   /**
@@ -337,8 +352,8 @@ export class Pipeline implements PipelineAccessor {
    * Updates the timeframe clock after the message has bee processed.
    */
   async *consume(): AsyncIterable<FeedMessageBlock> {
-    invariant(!this._isOpen, 'Pipeline is already being consumed.');
-    this._isOpen = true;
+    invariant(!this._isBeingConsumed, 'Pipeline is already being consumed.');
+    this._isBeingConsumed = true;
 
     invariant(this._feedSetIterator, 'Iterator not initialized.');
     let lastFeedSetIterator = this._feedSetIterator;
@@ -367,17 +382,27 @@ export class Pipeline implements PipelineAccessor {
     }
 
     // TODO(burdon): Test re-entrant?
-    this._isOpen = false;
+    this._isBeingConsumed = false;
   }
 
   private _setFeedDownloadState(feed: FeedWrapper<FeedMessage>) {
-    const timeframe = this._state._startTimeframe;
-    const seq = timeframe.get(feed.key) ?? 0;
+    let handle = this._downloads.get(feed); // TODO(burdon): Always undefined.
+    if (handle) {
+      feed.undownload(handle);
+    }
 
-    feed.undownload({ callback: () => log('undownload') });
-    feed.download({ start: seq + 1, linear: true }).catch((err: Error) => {
-      log('failed to download feed', { err });
+    const timeframe = this._state._startTimeframe;
+    const seq = timeframe.get(feed.key) ?? -1;
+    log.info('download', { feed: feed.key.truncate(), seq, length: feed.length });
+    handle = feed.download({ start: seq + 1, linear: true }, (err: any, data: any) => {
+      if (err) {
+        // log.error(err); // TODO(burdon): Feed is closed.
+      } else {
+        log.info('data', data); // TODO(burdon): Never called.
+      }
     });
+
+    this._downloads.set(feed, handle);
   }
 
   private async _initIterator() {

package/src/space/control-pipeline.ts

@@ -2,13 +2,16 @@
 // Copyright 2022 DXOS.org
 //
 
+import { Context } from '@dxos/context';
 import { SpaceStateMachine, SpaceState, MemberInfo, FeedInfo } from '@dxos/credentials';
 import { FeedWrapper } from '@dxos/feed-store';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
+import { FeedMessageBlock } from '@dxos/protocols';
 import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { AdmittedFeed, Credential } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { Timeframe } from '@dxos/timeframe';
+import { TimeSeriesCounter, TimeUsageCounter, trace } from '@dxos/tracing';
 import { AsyncCallback, Callback, tracer } from '@dxos/util';
 
 import { MetadataStore } from '../metadata';
@@ -26,6 +29,7 @@ const TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
 /**
  * Processes HALO credentials, which include genesis and invitations.
  */
+@trace.resource()
 export class ControlPipeline {
   private readonly _pipeline: Pipeline;
   private readonly _spaceStateMachine: SpaceStateMachine;
@@ -39,6 +43,12 @@ export class ControlPipeline {
   public readonly onMemberAdmitted: Callback<AsyncCallback<MemberInfo>>;
   public readonly onCredentialProcessed: Callback<AsyncCallback<Credential>>;
 
+  @trace.metricsCounter()
+  private _usage = new TimeUsageCounter();
+
+  @trace.metricsCounter()
+  private _mutations = new TimeSeriesCounter();
+
   constructor({ spaceKey, genesisFeed, feedProvider, metadataStore }: ControlPipelineParams) {
     this._spaceKey = spaceKey;
     this._metadata = metadataStore;
@@ -82,34 +92,49 @@ export class ControlPipeline {
   async start() {
     log('starting...');
     setTimeout(async () => {
-      for await (const msg of this._pipeline.consume()) {
-        try {
-          // log('processing', { msg });
-          log('processing', { key: msg.feedKey, seq: msg.seq });
-          if (msg.data.payload.credential) {
-            const timer = tracer.mark('dxos.echo.pipeline.control');
-            const result = await this._spaceStateMachine.process(
-              msg.data.payload.credential.credential,
-              PublicKey.from(msg.feedKey),
-            );
-
-            timer.end();
-            if (!result) {
-              log.warn('processing failed', { msg });
-            } else {
-              await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
-            }
-          }
-        } catch (err: any) {
-          log.catch(err);
-        }
-      }
+      void this._consumePipeline(new Context());
     });
 
     await this._pipeline.start();
     log('started');
   }
 
+  @trace.span()
+  private async _consumePipeline(ctx: Context) {
+    for await (const msg of this._pipeline.consume()) {
+      const span = this._usage.beginRecording();
+      this._mutations.inc();
+
+      try {
+        await this._processMessage(ctx, msg);
+      } catch (err: any) {
+        log.catch(err);
+      }
+
+      span.end();
+    }
+  }
+
+  @trace.span()
+  private async _processMessage(ctx: Context, msg: FeedMessageBlock) {
+    // log('processing', { msg });
+    log('processing', { key: msg.feedKey, seq: msg.seq });
+    if (msg.data.payload.credential) {
+      const timer = tracer.mark('dxos.echo.pipeline.control');
+      const result = await this._spaceStateMachine.process(
+        msg.data.payload.credential.credential,
+        PublicKey.from(msg.feedKey),
+      );
+
+      timer.end();
+      if (!result) {
+        log.warn('processing failed', { msg });
+      } else {
+        await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
+      }
+    }
+  }
+
   private async _noteTargetStateIfNeeded(timeframe: Timeframe) {
     // TODO(dmaretskyi): Replace this with a proper debounce/throttle.
 

package/src/space/data-pipeline.ts

@@ -5,12 +5,12 @@
 import { Event, scheduleTask, synchronized, trackLeaks } from '@dxos/async';
 import { Context } from '@dxos/context';
 import { CredentialProcessor, FeedInfo, SpecificCredential, checkCredentialType } from '@dxos/credentials';
-import { getStateMachineFromItem, ItemManager } from '@dxos/echo-db';
+import { getStateMachineFromItem, ItemManager, TYPE_PROPERTIES } from '@dxos/echo-db';
 import { CancelledError } from '@dxos/errors';
 import { FeedWriter } from '@dxos/feed-store';
 import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
-import { log } from '@dxos/log';
+import { log, omit } from '@dxos/log';
 import { ModelFactory } from '@dxos/model-factory';
 import { DataPipelineProcessed } from '@dxos/protocols';
 import { DataMessage } from '@dxos/protocols/proto/dxos/echo/feed';
@@ -19,6 +19,7 @@ import { ObjectSnapshot } from '@dxos/protocols/proto/dxos/echo/model/document';
 import { SpaceSnapshot } from '@dxos/protocols/proto/dxos/echo/snapshot';
 import { Credential, Epoch } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { Timeframe } from '@dxos/timeframe';
+import { TimeSeriesCounter, TimeUsageCounter, trace } from '@dxos/tracing';
 import { tracer } from '@dxos/util';
 
 import { DatabaseHost, SnapshotManager } from '../db-host';
@@ -65,6 +66,7 @@ const TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
  * Reacts to new epochs to restart the pipeline.
  */
 @trackLeaks('open', 'close')
+@trace.resource()
 export class DataPipeline implements CredentialProcessor {
   private _ctx = new Context();
   private _pipeline?: Pipeline = undefined;
@@ -78,6 +80,12 @@ export class DataPipeline implements CredentialProcessor {
   private _lastProcessedEpoch = -1;
   private _epochCtx?: Context;
 
+  @trace.metricsCounter()
+  private _usage = new TimeUsageCounter();
+
+  @trace.metricsCounter()
+  private _mutations = new TimeSeriesCounter();
+
   constructor(private readonly _params: DataPipelineParams) {}
 
   public itemManager!: ItemManager;
@@ -132,8 +140,9 @@ export class DataPipeline implements CredentialProcessor {
 
     this._pipeline = new Pipeline();
     await this._params.onPipelineCreated(this._pipeline);
-    await this._pipeline.start();
+
     await this._pipeline.pause(); // Start paused until we have the first epoch.
+    await this._pipeline.start();
 
     if (this._targetTimeframe) {
       this._pipeline.state.setTargetTimeframe(this._targetTimeframe);
@@ -205,6 +214,9 @@ export class DataPipeline implements CredentialProcessor {
 
     invariant(this._pipeline, 'Pipeline is not initialized.');
     for await (const msg of this._pipeline.consume()) {
+      const span = this._usage.beginRecording();
+      this._mutations.inc();
+
       const { feedKey, seq, data } = msg;
       log('processing message', { feedKey, seq });
 
@@ -241,6 +253,8 @@ export class DataPipeline implements CredentialProcessor {
       } catch (err: any) {
         log.catch(err);
       }
+
+      span.end();
     }
   }
 
@@ -268,7 +282,7 @@ export class DataPipeline implements CredentialProcessor {
         (item) =>
           item.modelMeta?.type === 'dxos:model/document' &&
           // TODO(burdon): Document?
-          (getStateMachineFromItem(item)?.snapshot() as ObjectSnapshot).type === 'dxos.sdk.client.Properties',
+          (getStateMachineFromItem(item)?.snapshot() as ObjectSnapshot).type === TYPE_PROPERTIES,
       );
       if (propertiesItem) {
         cache.properties = getStateMachineFromItem(propertiesItem)?.snapshot() as ObjectSnapshot;
@@ -318,7 +332,6 @@ export class DataPipeline implements CredentialProcessor {
         // Space closed before we got to process the epoch.
         return;
       }
-      log('process epoch', { epoch });
       await this._processEpoch(ctx, epoch.subject.assertion);
 
       this.appliedEpoch = epoch;
@@ -332,16 +345,13 @@ export class DataPipeline implements CredentialProcessor {
     invariant(this._pipeline);
     this._lastProcessedEpoch = epoch.number;
 
-    log('Processing epoch', { epoch });
+    log('processing', { epoch: omit(epoch, 'proof') });
     if (epoch.snapshotCid) {
       const snapshot = await this._params.snapshotManager.load(ctx, epoch.snapshotCid);
-
-      // TODO(dmaretskyi): Clearing old items + events.
       this.databaseHost!._itemDemuxer.restoreFromSnapshot(snapshot.database);
     }
 
-    log('restarting pipeline for epoch');
-
+    log('restarting pipeline from epoch');
     await this._pipeline.pause();
     await this._pipeline.setCursor(epoch.timeframe);
     await this._pipeline.unpause();

package/src/space/space-manager.ts

@@ -122,6 +122,7 @@ export class SpaceManager {
       memberKey,
     });
     this._spaces.set(space.key, space);
+
     log.trace('dxos.echo.space-manager.construct-space', trace.end({ id: this._instanceId }));
     return space;
   }

package/src/space/space-protocol.ts

@@ -87,6 +87,7 @@ export class SpaceProtocol {
     this._onAuthFailure = onAuthFailure;
     this.blobSync = new BlobSync({ blobStore });
 
+    // TODO(burdon): Async race condition? Move to start?
     this._topic = subtleCrypto.digest('SHA-256', topic.asBuffer()).then(discoveryKey).then(PublicKey.from);
   }
 
@@ -100,6 +101,7 @@ export class SpaceProtocol {
     }
   }
 
+  // TODO(burdon): Rename open? Common open/close interfaces for all services?
   async start() {
     if (this._connection) {
       return;
@@ -124,7 +126,7 @@ export class SpaceProtocol {
       peerId: this._swarmIdentity.peerKey,
       topic,
       topology: new MMSTTopology(topologyConfig),
-      label: `Protocol swarm: ${topic}`,
+      label: `space swarm ${topic.truncate()}`,
     });
 
     log('started');

package/src/space/space.ts

@@ -140,6 +140,7 @@ export class Space {
   }
 
   @logInfo
+  @trace.info()
   get key() {
     return this._key;
   }