@dxos/echo-pipeline 0.1.56-main.e47dfd1 → 0.1.56-main.e79d64a

package/src/pipeline/pipeline-stress.test.ts

@@ -0,0 +1,226 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import expect from 'expect';
+import * as fc from 'fast-check';
+import { inspect } from 'util';
+
+import { asyncTimeout } from '@dxos/async';
+import { FeedStore, FeedWrapper } from '@dxos/feed-store';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { FeedMessageBlock } from '@dxos/protocols';
+import { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+import { describe, test } from '@dxos/test';
+import { Timeframe } from '@dxos/timeframe';
+import { range } from '@dxos/util';
+
+import { TestFeedBuilder } from '../testing';
+import { Pipeline } from './pipeline';
+
+const NUM_AGENTS = 2;
+const NUM_MESSAGES = 10;
+
+// TODO(burdon): Describe test.
+describe('pipeline/stress test', () => {
+  test
+    .skip('stress', async () => {
+      const builder = new TestFeedBuilder();
+
+      const agentIds = range(NUM_AGENTS).map(() => PublicKey.random().toHex().slice(0, 8));
+      const anAgentId = fc.constantFrom(...agentIds);
+
+      const commands = fc.commands(
+        [
+          fc.tuple(anAgentId, fc.integer({ min: 1, max: 10 })).map(([agent, count]) => new WriteCommand(agent, count)),
+          fc.constant(new SyncCommand()),
+          anAgentId.map((agent) => new RestartCommand(agent)),
+        ],
+        { size: 'large' },
+      );
+
+      const model = fc.asyncProperty(commands, async (commands) => {
+        const feedStore = builder.createFeedStore();
+
+        const agents = new Map(agentIds.map((id) => [id, new Agent(builder, feedStore, id)]));
+        await Promise.all(Array.from(agents.values()).map((agent) => agent.open()));
+        await Promise.all(Array.from(agents.values()).map((agent) => agent.start()));
+
+        const setup: fc.ModelRunSetup<Model, Real> = () => ({
+          model: {},
+          real: {
+            feedStore,
+            agents,
+          },
+        });
+
+        try {
+          await fc.asyncModelRun(setup, [...commands, new SyncCommand()]);
+        } finally {
+          await Promise.all(Array.from(agents.values()).map((agent) => agent.stop()));
+          await Promise.all(Array.from(agents.values()).map((agent) => agent.close()));
+        }
+      });
+
+      const examples: [commands: Iterable<fc.AsyncCommand<Model, Real, boolean>>][] = [
+        [[new WriteCommand(agentIds[0], 10), new WriteCommand(agentIds[1], 10), new SyncCommand()]],
+        [[new WriteCommand(agentIds[0], 4), new RestartCommand(agentIds[0]), new SyncCommand()]],
+      ];
+
+      await fc.assert(model, { examples });
+    })
+    .timeout(60_000);
+});
+
+class Agent {
+  public startingTimeframe = new Timeframe();
+  public pipeline!: Pipeline;
+  public feed!: FeedWrapper<FeedMessage>;
+  public messages: FeedMessageBlock[] = [];
+  public writePromise: Promise<any> = Promise.resolve();
+
+  constructor(private readonly builder: TestFeedBuilder, public feedStore: FeedStore<FeedMessage>, public id: string) {}
+
+  async open() {
+    const key = await this.builder.keyring.createKey();
+    this.feed = await this.feedStore.openFeed(key, { writable: true });
+  }
+
+  async start() {
+    this.pipeline = new Pipeline();
+    await this.pipeline.setCursor(this.startingTimeframe);
+    await this.pipeline.start();
+
+    // NOTE: not awaiting here breaks the test.
+    await Promise.all(this.feedStore.feeds.map((feed) => this.pipeline.addFeed(feed)));
+    this.pipeline.setWriteFeed(this.feed);
+
+    // consume in async task.
+    void this.consume();
+  }
+
+  async stop() {
+    await this.pipeline.stop();
+    this.startingTimeframe = this.pipeline.state.timeframe;
+  }
+
+  async close() {
+    await this.feed.close();
+  }
+
+  write(message: FeedMessage.Payload) {
+    const prev = this.writePromise;
+    const promise = this.pipeline.writer!.write(message);
+    this.writePromise = Promise.all([prev, promise]);
+  }
+
+  async consume() {
+    for await (const msg of this.pipeline.consume()) {
+      this.messages.push(msg);
+    }
+
+    log('stopped consuming');
+  }
+}
+
+type Model = {};
+type Real = {
+  feedStore: FeedStore<FeedMessage>;
+  agents: Map<string, Agent>;
+};
+
+class WriteCommand implements fc.AsyncCommand<Model, Real> {
+  constructor(public agent: string, public count: number) {}
+
+  check = () => true;
+
+  async run(model: Model, real: Real) {
+    // console.log(`WriteCommand(${this.agent}, ${this.count})`);
+    const agent = real.agents.get(this.agent)!;
+    const toWrite = Math.min(this.count, NUM_MESSAGES - agent.feed.length);
+    if (toWrite > 0) {
+      for (const _ of range(toWrite)) {
+        agent.write({}); // Content is not important.
+      }
+    }
+  }
+
+  toString = () => `WriteCommand(${this.agent}, ${this.count})`;
+}
+
+class SyncCommand implements fc.AsyncCommand<Model, Real> {
+  check = () => true;
+
+  async run(model: Model, real: Real) {
+    // console.log('SyncCommand()');
+    const targets: any = {};
+
+    try {
+      for (const agent of real.agents.values()) {
+        await agent.writePromise;
+      }
+
+      for (const agent of real.agents.values()) {
+        targets[agent.id] = agent.pipeline.state.endTimeframe;
+        if (agent.pipeline.state.endTimeframe.isEmpty()) {
+          log('empty endtimeframe', {
+            id: agent.id,
+            endTimeframe: agent.pipeline.state.endTimeframe,
+            feeds: agent.pipeline.getFeeds().map((feed) => [feed.key.toString(), feed.length]),
+          });
+        }
+        await asyncTimeout(agent.pipeline.state.waitUntilTimeframe(agent.pipeline.state.endTimeframe), 1000);
+      }
+
+      const tf: Timeframe = real.agents.values().next().value!.pipeline.state.timeframe;
+      for (const agent of real.agents.values()) {
+        expect(agent.pipeline.state.timeframe.equals(tf)).toEqual(true);
+      }
+
+      const totalMessages = real.feedStore.feeds.reduce((acc, feed) => acc + feed.length, 0);
+      for (const agent of real.agents.values()) {
+        expect(agent.messages.length).toEqual(totalMessages);
+      }
+    } catch (err) {
+      log(
+        inspect(
+          {
+            agents: Array.from(real.agents.values()).map((agent) => ({
+              id: agent.id,
+              messages: agent.messages.length,
+              feeds: agent.pipeline.getFeeds().map((feed) => [feed.key, feed.length]),
+              timeframe: agent.pipeline.state.timeframe,
+              endTimeframe: agent.pipeline.state.endTimeframe,
+            })),
+            feeds: real.feedStore.feeds.map((feed) => [feed.key, feed.length]),
+            targets,
+          },
+          false,
+          null,
+          true,
+        ),
+      );
+      throw err;
+    }
+  }
+
+  toString = () => 'SyncCommand()';
+}
+
+class RestartCommand implements fc.AsyncCommand<Model, Real> {
+  constructor(public agent: string) {}
+
+  check = () => true;
+
+  async run(model: Model, real: Real) {
+    log(`RestartCommand(${this.agent})`);
+
+    const agent = real.agents.get(this.agent)!;
+    await agent.writePromise;
+    await agent.stop();
+    await agent.start();
+  }
+
+  toString = () => `RestartCommand(${this.agent})`;
+}

package/src/pipeline/pipeline.test.ts

@@ -3,14 +3,8 @@
 //
 
 import expect from 'expect';
-import * as fc from 'fast-check';
-import { inspect } from 'util';
 
-import { asyncTimeout } from '@dxos/async';
-import { FeedStore, FeedWrapper } from '@dxos/feed-store';
-import { PublicKey } from '@dxos/keys';
-import { log } from '@dxos/log';
-import { FeedMessageBlock } from '@dxos/protocols';
+import { Event, sleep } from '@dxos/async';
 import { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { describe, test, afterTest } from '@dxos/test';
 import { Timeframe } from '@dxos/timeframe';
@@ -85,7 +79,7 @@ describe('pipeline/Pipeline', () => {
 
     const processedSequenceNumbers: number[] = [];
 
-    // skip first 10, process the rest, and the repeat the last 10.
+    // Skip first 10, process the rest, and the repeat the last 10.
     const expectedSequenceNumbers = [...sequenceNumbers.slice(10, 30), ...sequenceNumbers.slice(20, 30)];
 
     await pipeline.setCursor(new Timeframe([[feed.key, 9]]));
@@ -109,209 +103,39 @@ describe('pipeline/Pipeline', () => {
     expect(processedSequenceNumbers).toEqual(expectedSequenceNumbers);
   });
 
-  test
-    .skip('stress', async () => {
-      const builder = new TestFeedBuilder();
-
-      const NUM_AGENTS = 2;
-      const NUM_MESSAGES = 100;
-
-      class Agent {
-        public startingTimeframe = new Timeframe();
-        public pipeline!: Pipeline;
-        public feed!: FeedWrapper<FeedMessage>;
-        public messages: FeedMessageBlock[] = [];
-        public writePromise: Promise<any> = Promise.resolve();
-
-        constructor(public id: string, public feedStore: FeedStore<FeedMessage>) {}
-
-        async open() {
-          const key = await builder.keyring.createKey();
-          this.feed = await this.feedStore.openFeed(key, { writable: true });
-        }
-
-        async start() {
-          this.pipeline = new Pipeline();
-          await this.pipeline.setCursor(this.startingTimeframe);
-
-          await this.pipeline.start();
-
-          // NOTE: not awaiting here breaks the test.
-          await Promise.all(this.feedStore.feeds.map((feed) => this.pipeline.addFeed(feed)));
-          this.pipeline.setWriteFeed(this.feed);
-
-          // consume in async task.
-          void this.consume();
-        }
-
-        async stop() {
-          await this.pipeline.stop();
-          this.startingTimeframe = this.pipeline.state.timeframe;
-        }
-
-        async close() {
-          await this.feed.close();
-        }
-
-        write(message: FeedMessage.Payload) {
-          const prev = this.writePromise;
-          const promise = this.pipeline.writer!.write(message);
-          this.writePromise = Promise.all([prev, promise]);
-        }
-
-        async consume() {
-          for await (const msg of this.pipeline.consume()) {
-            this.messages.push(msg);
-          }
-          log('stopped consuming');
-        }
-      }
-
-      type Model = {};
-      type Real = {
-        feedStore: FeedStore<FeedMessage>;
-        agents: Map<string, Agent>;
-      };
-
-      class WriteCommand implements fc.AsyncCommand<Model, Real> {
-        constructor(public agent: string, public count: number) {}
-
-        check = () => true;
-
-        async run(model: Model, real: Real) {
-          console.log(`WriteCommand(${this.agent}, ${this.count})`);
-
-          const agent = real.agents.get(this.agent)!;
-
-          const toWrite = Math.min(this.count, NUM_MESSAGES - agent.feed.length);
-          if (toWrite > 0) {
-            for (const _ of range(toWrite)) {
-              agent.write({}); // Content is not important.
-            }
-          }
-        }
-
-        toString = () => `WriteCommand(${this.agent}, ${this.count})`;
-      }
-
-      class SyncCommand implements fc.AsyncCommand<Model, Real> {
-        check = () => true;
-
-        async run(model: Model, real: Real) {
-          console.log('SyncCommand()');
-          const targets: any = {};
-
-          try {
-            for (const agent of real.agents.values()) {
-              await agent.writePromise;
-            }
-
-            for (const agent of real.agents.values()) {
-              targets[agent.id] = agent.pipeline.state.endTimeframe;
-              if (agent.pipeline.state.endTimeframe.isEmpty()) {
-                console.log('empty endtimeframe', {
-                  id: agent.id,
-                  endTimeframe: agent.pipeline.state.endTimeframe,
-                  feeds: agent.pipeline.getFeeds().map((feed) => [feed.key.toString(), feed.length]),
-                });
-              }
-              await asyncTimeout(agent.pipeline.state.waitUntilTimeframe(agent.pipeline.state.endTimeframe), 1000);
-            }
-
-            const tf: Timeframe = real.agents.values().next().value!.pipeline.state.timeframe;
-            for (const agent of real.agents.values()) {
-              expect(agent.pipeline.state.timeframe.equals(tf)).toEqual(true);
-            }
-
-            const totalMessages = real.feedStore.feeds.reduce((acc, feed) => acc + feed.length, 0);
-            for (const agent of real.agents.values()) {
-              expect(agent.messages.length).toEqual(totalMessages);
-            }
-          } catch (err) {
-            console.log(
-              inspect(
-                {
-                  agents: Array.from(real.agents.values()).map((agent) => ({
-                    id: agent.id,
-                    messages: agent.messages.length,
-                    feeds: agent.pipeline.getFeeds().map((feed) => [feed.key, feed.length]),
-                    timeframe: agent.pipeline.state.timeframe,
-                    endTimeframe: agent.pipeline.state.endTimeframe,
-                  })),
-                  feeds: real.feedStore.feeds.map((feed) => [feed.key, feed.length]),
-                  targets,
-                },
-                false,
-                null,
-                true,
-              ),
-            );
-            throw err;
-          }
-        }
-
-        toString = () => 'SyncCommand()';
-      }
-
-      class RestartCommand implements fc.AsyncCommand<Model, Real> {
-        constructor(public agent: string) {}
-
-        check = () => true;
-
-        async run(model: Model, real: Real) {
-          console.log(`RestartCommand(${this.agent})`);
+  test('cursor change while polling', async () => {
+    const pipeline = new Pipeline();
+    afterTest(() => pipeline.stop());
 
-          const agent = real.agents.get(this.agent)!;
+    const builder = new TestFeedBuilder();
+    const feedStore = builder.createFeedStore();
+    const key = await builder.keyring.createKey();
+    const feed = await feedStore.openFeed(key, { writable: true });
+    await pipeline.addFeed(feed);
 
-          await agent.writePromise;
-          await agent.stop();
-          await agent.start();
-        }
+    for (const _ of range(20)) {
+      await feed.appendWithReceipt(TEST_MESSAGE);
+    }
+    await feed.clear(0, 10);
 
-        toString = () => `RestartCommand(${this.agent})`;
+    const processedSequenceNumbers: number[] = [];
+    const expectedSequenceNumbers = range(20).slice(10);
+    const processedEvent = new Event();
+    setTimeout(async () => {
+      for await (const block of pipeline.consume()) {
+        processedSequenceNumbers.push(block.seq);
+        processedEvent.emit();
       }
+    });
 
-      const agentIds = range(NUM_AGENTS).map(() => PublicKey.random().toHex().slice(0, 8));
-      const anAgentId = fc.constantFrom(...agentIds);
-
-      const commands = fc.commands(
-        [
-          fc.tuple(anAgentId, fc.integer({ min: 1, max: 10 })).map(([agent, count]) => new WriteCommand(agent, count)),
-          fc.constant(new SyncCommand()),
-          anAgentId.map((agent) => new RestartCommand(agent)),
-        ],
-        { size: 'large' },
-      );
-
-      const model = fc.asyncProperty(commands, async (commands) => {
-        const feedStore = builder.createFeedStore();
-
-        const agents = new Map(agentIds.map((id) => [id, new Agent(id, feedStore)]));
-        await Promise.all(Array.from(agents.values()).map((agent) => agent.open()));
-        await Promise.all(Array.from(agents.values()).map((agent) => agent.start()));
-
-        const setup: fc.ModelRunSetup<Model, Real> = () => ({
-          model: {},
-          real: {
-            feedStore,
-            agents,
-          },
-        });
-
-        try {
-          await fc.asyncModelRun(setup, [...commands, new SyncCommand()]);
-        } finally {
-          await Promise.all(Array.from(agents.values()).map((agent) => agent.stop()));
-          await Promise.all(Array.from(agents.values()).map((agent) => agent.close()));
-        }
-      });
+    await pipeline.start();
+    await sleep(1000);
 
-      const examples: [commands: Iterable<fc.AsyncCommand<Model, Real, boolean>>][] = [
-        [[new WriteCommand(agentIds[0], 10), new WriteCommand(agentIds[1], 10), new SyncCommand()]],
-        [[new WriteCommand(agentIds[0], 4), new RestartCommand(agentIds[0]), new SyncCommand()]],
-      ];
+    await pipeline.pause();
+    await pipeline.setCursor(new Timeframe([[feed.key, 9]]));
+    await pipeline.unpause();
 
-      await fc.assert(model, { examples });
-    })
-    .timeout(60_000);
+    await processedEvent.waitForCondition(() => processedSequenceNumbers.length === 10);
+    expect(processedSequenceNumbers).toEqual(expectedSequenceNumbers);
+  });
 });

package/src/pipeline/pipeline.ts

@@ -2,12 +2,11 @@
 // Copyright 2022 DXOS.org
 //
 
-import invariant from 'tiny-invariant';
-
 import { Event, sleep, synchronized, Trigger } from '@dxos/async';
 import { Context, rejectOnDispose } from '@dxos/context';
 import { failUndefined } from '@dxos/debug';
 import { FeedSetIterator, FeedWrapper, FeedWriter } from '@dxos/feed-store';
+import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 import { FeedMessageBlock } from '@dxos/protocols';
@@ -205,9 +204,6 @@ export class Pipeline implements PipelineAccessor {
   private readonly _timeframeClock = new TimeframeClock(new Timeframe());
   private readonly _feeds = new ComplexMap<PublicKey, FeedWrapper<FeedMessage>>(PublicKey.hash);
 
-  // Inbound feed stream.
-  private _feedSetIterator?: FeedSetIterator<FeedMessage>;
-
   // External state accessor.
   private readonly _state: PipelineState = new PipelineState(this._feeds, this._timeframeClock);
 
@@ -215,14 +211,20 @@ export class Pipeline implements PipelineAccessor {
   private readonly _processingTrigger = new Trigger().wake();
   private readonly _pauseTrigger = new Trigger().wake();
 
-  private _isStopping = false;
-  private _isStarted = false;
-  private _isOpen = false;
-  private _isPaused = false;
+  // Pending downloads.
+  private readonly _downloads = new ComplexMap<FeedWrapper<FeedMessage>, any>((value) => PublicKey.hash(value.key));
+
+  // Inbound feed stream.
+  private _feedSetIterator?: FeedSetIterator<FeedMessage>;
 
   // Outbound feed writer.
   private _writer: FeedWriter<FeedMessage.Payload> | undefined;
 
+  private _isStopping = false;
+  private _isStarted = false;
+  private _isBeingConsumed = false;
+  private _isPaused = false;
+
   get state() {
     return this._state;
   }
@@ -244,10 +246,14 @@ export class Pipeline implements PipelineAccessor {
   // which might be opening feeds during the mutation processing, which w
   async addFeed(feed: FeedWrapper<FeedMessage>) {
     this._feeds.set(feed.key, feed);
+
     if (this._feedSetIterator) {
       await this._feedSetIterator.addFeed(feed);
     }
-    this._setFeedDownloadState(feed);
+
+    if (this._isStarted && !this._isPaused) {
+      this._setFeedDownloadState(feed);
+    }
   }
 
   setWriteFeed(feed: FeedWrapper<FeedMessage>) {
@@ -265,17 +271,28 @@ export class Pipeline implements PipelineAccessor {
 
   @synchronized
   async start() {
+    invariant(!this._isStarted, 'Pipeline is already started.');
     log('starting...');
     await this._initIterator();
     await this._feedSetIterator!.open();
     this._isStarted = true;
     log('started');
+
+    if (!this._isPaused) {
+      for (const feed of this._feeds.values()) {
+        this._setFeedDownloadState(feed);
+      }
+    }
   }
 
   @synchronized
   async stop() {
     log('stopping...');
     this._isStopping = true;
+    for (const [feed, handle] of this._downloads.entries()) {
+      feed.undownload(handle);
+    }
+    this._downloads.clear();
     await this._feedSetIterator?.close();
     await this._processingTrigger.wait(); // Wait for the in-flight message to be processed.
     await this._state._ctx.dispose();
@@ -297,10 +314,6 @@ export class Pipeline implements PipelineAccessor {
     this._timeframeClock.setTimeframe(timeframe);
 
     // Cancel downloads of mutations before the cursor.
-    for (const feed of this._feeds.values()) {
-      this._setFeedDownloadState(feed);
-    }
-
     if (this._feedSetIterator) {
       await this._feedSetIterator.close();
       await this._initIterator();
@@ -313,7 +326,6 @@ export class Pipeline implements PipelineAccessor {
    */
   @synchronized
   async pause() {
-    invariant(this._isStarted, 'Pipeline is not open.');
     if (this._isPaused) {
       return;
     }
@@ -325,11 +337,14 @@ export class Pipeline implements PipelineAccessor {
 
   @synchronized
   async unpause() {
-    invariant(this._isStarted, 'Pipeline is not open.');
     invariant(this._isPaused, 'Pipeline is not paused.');
 
     this._pauseTrigger.wake();
     this._isPaused = false;
+
+    for (const feed of this._feeds.values()) {
+      this._setFeedDownloadState(feed);
+    }
   }
 
   /**
@@ -337,8 +352,8 @@ export class Pipeline implements PipelineAccessor {
    * Updates the timeframe clock after the message has bee processed.
    */
   async *consume(): AsyncIterable<FeedMessageBlock> {
-    invariant(!this._isOpen, 'Pipeline is already being consumed.');
-    this._isOpen = true;
+    invariant(!this._isBeingConsumed, 'Pipeline is already being consumed.');
+    this._isBeingConsumed = true;
 
     invariant(this._feedSetIterator, 'Iterator not initialized.');
     let lastFeedSetIterator = this._feedSetIterator;
@@ -354,6 +369,7 @@ export class Pipeline implements PipelineAccessor {
         iterable = lastFeedSetIterator[Symbol.asyncIterator]();
       }
 
+      // Will be canceled when the iterator gets closed.
       const { done, value } = await iterable.next();
       if (!done) {
         const block = value ?? failUndefined();
@@ -366,17 +382,27 @@ export class Pipeline implements PipelineAccessor {
     }
 
     // TODO(burdon): Test re-entrant?
-    this._isOpen = false;
+    this._isBeingConsumed = false;
   }
 
   private _setFeedDownloadState(feed: FeedWrapper<FeedMessage>) {
-    const timeframe = this._state._startTimeframe;
-    const seq = timeframe.get(feed.key) ?? 0;
+    let handle = this._downloads.get(feed); // TODO(burdon): Always undefined.
+    if (handle) {
+      feed.undownload(handle);
+    }
 
-    feed.undownload({ callback: () => log('undownload') });
-    feed.download({ start: seq + 1, linear: true }).catch((err: Error) => {
-      log('failed to download feed', { err });
+    const timeframe = this._state._startTimeframe;
+    const seq = timeframe.get(feed.key) ?? -1;
+    log.info('download', { feed: feed.key.truncate(), seq, length: feed.length });
+    handle = feed.download({ start: seq + 1, linear: true }, (err: any, data: any) => {
+      if (err) {
+        // log.error(err); // TODO(burdon): Feed is closed.
+      } else {
+        log.info('data', data); // TODO(burdon): Never called.
+      }
     });
+
+    this._downloads.set(feed, handle);
   }
 
   private async _initIterator() {

package/src/space/auth.ts

@@ -2,11 +2,10 @@
 // Copyright 2019 DXOS.org
 //
 
-import invariant from 'tiny-invariant';
-
 import { runInContext, scheduleTask } from '@dxos/async';
 import { Context } from '@dxos/context';
 import { randomBytes } from '@dxos/crypto';
+import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 import { schema } from '@dxos/protocols';
 import { AuthService } from '@dxos/protocols/proto/dxos/mesh/teleport/auth';