@dxos/client-services 0.10.0 → 0.11.1

package/src/packlets/pipeline/feeds.ts

@@ -0,0 +1,22 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { type FeedWriter } from '@dxos/feed-store';
+import { invariant } from '@dxos/invariant';
+import { type MaybePromise } from '@dxos/util';
+
+/**
+ * Maps the written arguments onto a different message type.
+ */
+export const createMappedFeedWriter = <Source extends {}, Target extends {}>(
+  mapper: (arg: Source) => MaybePromise<Target>,
+  writer: FeedWriter<Target>,
+): FeedWriter<Source> => {
+  invariant(mapper);
+  invariant(writer);
+
+  return {
+    write: async (data: Source, options) => await writer.write(await mapper(data), options),
+  };
+};

package/src/packlets/pipeline/index.ts

@@ -0,0 +1,8 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+export { Pipeline, type PipelineAccessor } from './pipeline';
+export * from './timeframe-clock';
+export * from './codec';
+export * from './feeds';

package/src/packlets/pipeline/message-selector.ts

@@ -0,0 +1,35 @@
+//
+// Copyright 2020 DXOS.org
+//
+
+import { type FeedBlock, type FeedBlockSelector } from '@dxos/feed-store';
+import { invariant } from '@dxos/invariant';
+import { log } from '@dxos/log';
+import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+
+import { type TimeframeClock } from './timeframe-clock';
+
+/**
+ * The MessageSelector makes sure that we read in a trusted order.
+ * The first message we wish to process is the SpaceGenesis, which will admit a Feed.
+ * As we encounter and process FeedAdmit messages those are added to the Space's trust,
+ * and we begin processing messages from them as well.
+ */
+export const createMessageSelector = (timeframeClock: TimeframeClock): FeedBlockSelector<FeedMessage> => {
+  return (messages: FeedBlock<FeedMessage>[]) => {
+    // Pick the first candidate with a valid timeframe that has no gaps.
+    for (let i = 0; i < messages.length; i++) {
+      const {
+        data: { timeframe },
+      } = messages[i];
+      invariant(timeframe);
+
+      if (!timeframeClock.hasGaps(timeframe)) {
+        return i;
+      }
+    }
+
+    // Not ready for this message yet.
+    log('Skipping...');
+  };
+};

package/src/packlets/pipeline/pipeline-stress.test.ts

@@ -0,0 +1,230 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import * as fc from 'fast-check';
+import { inspect } from 'node:util';
+import { describe, expect, test } from 'vitest';
+
+import { asyncTimeout } from '@dxos/async';
+import { type FeedStore, type FeedWrapper } from '@dxos/feed-store';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { type FeedMessageBlock } from '@dxos/protocols';
+import { type FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+import { Timeframe } from '@dxos/timeframe';
+import { range } from '@dxos/util';
+
+import { Pipeline } from './pipeline';
+import { TestFeedBuilder } from './testing';
+
+const NUM_AGENTS = 2;
+const NUM_MESSAGES = 10;
+
+// TODO(burdon): Describe test.
+describe('pipeline/stress test', () => {
+  test.skip('stress', { timeout: 60_000 }, async () => {
+    const builder = new TestFeedBuilder();
+
+    const agentIds = range(NUM_AGENTS).map(() => PublicKey.random().toHex().slice(0, 8));
+    const anAgentId = fc.constantFrom(...agentIds);
+
+    const commands = fc.commands(
+      [
+        fc.tuple(anAgentId, fc.integer({ min: 1, max: 10 })).map(([agent, count]) => new WriteCommand(agent, count)),
+        fc.constant(new SyncCommand()),
+        anAgentId.map((agent) => new RestartCommand(agent)),
+      ],
+      { size: 'large' },
+    );
+
+    const model = fc.asyncProperty(commands, async (commands) => {
+      const feedStore = builder.createFeedStore();
+
+      const agents = new Map(agentIds.map((id) => [id, new Agent(builder, feedStore, id)]));
+      await Promise.all(Array.from(agents.values()).map((agent) => agent.open()));
+      await Promise.all(Array.from(agents.values()).map((agent) => agent.start()));
+
+      const setup: fc.ModelRunSetup<Model, Real> = () => ({
+        model: {},
+        real: {
+          feedStore,
+          agents,
+        },
+      });
+
+      try {
+        await fc.asyncModelRun(setup, [...commands, new SyncCommand()]);
+      } finally {
+        await Promise.all(Array.from(agents.values()).map((agent) => agent.stop()));
+        await Promise.all(Array.from(agents.values()).map((agent) => agent.close()));
+      }
+    });
+
+    const examples: [commands: Iterable<fc.AsyncCommand<Model, Real, boolean>>][] = [
+      [[new WriteCommand(agentIds[0], 10), new WriteCommand(agentIds[1], 10), new SyncCommand()]],
+      [[new WriteCommand(agentIds[0], 4), new RestartCommand(agentIds[0]), new SyncCommand()]],
+    ];
+
+    await fc.assert(model, { examples });
+  });
+});
+
+class Agent {
+  public startingTimeframe = new Timeframe();
+  public pipeline!: Pipeline;
+  public feed!: FeedWrapper<FeedMessage>;
+  public messages: FeedMessageBlock[] = [];
+  public writePromise: Promise<any> = Promise.resolve();
+
+  constructor(
+    private readonly builder: TestFeedBuilder,
+    public feedStore: FeedStore<FeedMessage>,
+    public id: string,
+  ) {}
+
+  async open(): Promise<void> {
+    const key = await this.builder.keyring.createKey();
+    this.feed = await this.feedStore.openFeed(key, { writable: true });
+  }
+
+  async start(): Promise<void> {
+    this.pipeline = new Pipeline();
+    await this.pipeline.setCursor(this.startingTimeframe);
+    await this.pipeline.start();
+
+    // NOTE: not awaiting here breaks the test.
+    await Promise.all(this.feedStore.feeds.map((feed) => this.pipeline.addFeed(feed)));
+    this.pipeline.setWriteFeed(this.feed);
+
+    // consume in async task.
+    void this.consume();
+  }
+
+  async stop(): Promise<void> {
+    await this.pipeline.stop();
+    this.startingTimeframe = this.pipeline.state.timeframe;
+  }
+
+  async close(): Promise<void> {
+    await this.feed.close();
+  }
+
+  write(message: FeedMessage.Payload): void {
+    const prev = this.writePromise;
+    const promise = this.pipeline.writer!.write(message);
+    this.writePromise = Promise.all([prev, promise]);
+  }
+
+  async consume(): Promise<void> {
+    for await (const msg of this.pipeline.consume()) {
+      this.messages.push(msg);
+    }
+
+    log('stopped consuming');
+  }
+}
+
+type Model = {};
+type Real = {
+  feedStore: FeedStore<FeedMessage>;
+  agents: Map<string, Agent>;
+};
+
+class WriteCommand implements fc.AsyncCommand<Model, Real> {
+  constructor(
+    public agent: string,
+    public count: number,
+  ) {}
+
+  check = () => true;
+
+  async run(model: Model, real: Real): Promise<void> {
+    // console.log(`WriteCommand(${this.agent}, ${this.count})`);
+    const agent = real.agents.get(this.agent)!;
+    const toWrite = Math.min(this.count, NUM_MESSAGES - agent.feed.length);
+    if (toWrite > 0) {
+      for (const _ of range(toWrite)) {
+        agent.write({}); // Content is not important.
+      }
+    }
+  }
+
+  toString = () => `WriteCommand(${this.agent}, ${this.count})`;
+}
+
+class SyncCommand implements fc.AsyncCommand<Model, Real> {
+  check = () => true;
+
+  async run(model: Model, real: Real): Promise<void> {
+    // console.log('SyncCommand()');
+    const targets: any = {};
+
+    try {
+      for (const agent of real.agents.values()) {
+        await agent.writePromise;
+      }
+
+      for (const agent of real.agents.values()) {
+        targets[agent.id] = agent.pipeline.state.endTimeframe;
+        if (agent.pipeline.state.endTimeframe.isEmpty()) {
+          log('empty endtimeframe', {
+            id: agent.id,
+            endTimeframe: agent.pipeline.state.endTimeframe,
+            feeds: agent.pipeline.getFeeds().map((feed) => [feed.key.toString(), feed.length]),
+          });
+        }
+        await asyncTimeout(agent.pipeline.state.waitUntilTimeframe(agent.pipeline.state.endTimeframe), 1000);
+      }
+
+      const tf: Timeframe = real.agents.values().next().value!.pipeline.state.timeframe;
+      for (const agent of real.agents.values()) {
+        expect(agent.pipeline.state.timeframe.equals(tf)).toEqual(true);
+      }
+
+      const totalMessages = real.feedStore.feeds.reduce((acc, feed) => acc + feed.length, 0);
+      for (const agent of real.agents.values()) {
+        expect(agent.messages.length).toEqual(totalMessages);
+      }
+    } catch (err) {
+      log(
+        inspect(
+          {
+            agents: Array.from(real.agents.values()).map((agent) => ({
+              id: agent.id,
+              messages: agent.messages.length,
+              feeds: agent.pipeline.getFeeds().map((feed) => [feed.key, feed.length]),
+              timeframe: agent.pipeline.state.timeframe,
+              endTimeframe: agent.pipeline.state.endTimeframe,
+            })),
+            feeds: real.feedStore.feeds.map((feed) => [feed.key, feed.length]),
+            targets,
+          },
+          false,
+          null,
+          true,
+        ),
+      );
+      throw err;
+    }
+  }
+
+  toString = () => 'SyncCommand()';
+}
+
+class RestartCommand implements fc.AsyncCommand<Model, Real> {
+  constructor(public agent: string) {}
+
+  check = () => true;
+
+  async run(model: Model, real: Real): Promise<void> {
+    log(`RestartCommand(${this.agent})`);
+
+    const agent = real.agents.get(this.agent)!;
+    await agent.writePromise;
+    await agent.stop();
+    await agent.start();
+  }
+
+  toString = () => `RestartCommand(${this.agent})`;
+}

package/src/packlets/pipeline/pipeline.test.ts

@@ -0,0 +1,140 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { describe, expect, onTestFinished, test } from 'vitest';
+
+import { Event, sleep } from '@dxos/async';
+import { type FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+import { Timeframe } from '@dxos/timeframe';
+import { range } from '@dxos/util';
+
+import { Pipeline } from './pipeline';
+import { TestFeedBuilder } from './testing';
+
+const TEST_MESSAGE: FeedMessage = {
+  timeframe: new Timeframe(),
+  payload: {},
+};
+
+describe('pipeline/Pipeline', () => {
+  test('asynchronous reader & writer without ordering', async () => {
+    const pipeline = new Pipeline();
+
+    const builder = new TestFeedBuilder();
+    const feedStore = builder.createFeedStore();
+
+    // Remote feeds from other peers.
+    const numFeeds = 5;
+    const messagesPerFeed = 10;
+    for (const _ in range(numFeeds)) {
+      const key = await builder.keyring.createKey();
+      const feed = await feedStore.openFeed(key, { writable: true });
+      void pipeline.addFeed(feed);
+
+      setTimeout(async () => {
+        for (const _ in range(messagesPerFeed)) {
+          await feed.append(TEST_MESSAGE);
+        }
+      });
+    }
+
+    // Local feed.
+    const key = await builder.keyring.createKey();
+    const feed = await feedStore.openFeed(key, { writable: true });
+    void pipeline.addFeed(feed);
+    pipeline.setWriteFeed(feed);
+
+    for (const _ in range(messagesPerFeed)) {
+      await pipeline.writer!.write({});
+    }
+
+    await pipeline.start();
+
+    let msgCount = 0;
+    for await (const _ of pipeline.consume()) {
+      if (++msgCount === numFeeds * messagesPerFeed) {
+        void pipeline.stop();
+      }
+    }
+  });
+
+  test('reading and writing with cursor changes', async () => {
+    const pipeline = new Pipeline();
+    onTestFinished(() => pipeline.stop());
+
+    const builder = new TestFeedBuilder();
+    const feedStore = builder.createFeedStore();
+    const key = await builder.keyring.createKey();
+    const feed = await feedStore.openFeed(key, { writable: true });
+    await pipeline.addFeed(feed);
+
+    const numMessages = 30;
+    const sequenceNumbers: number[] = [];
+    for (const _ of range(numMessages)) {
+      const { seq } = await feed.appendWithReceipt(TEST_MESSAGE);
+      sequenceNumbers.push(seq);
+    }
+
+    const processedSequenceNumbers: number[] = [];
+
+    // Skip first 10, process the rest, and the repeat the last 10.
+    const expectedSequenceNumbers = [...sequenceNumbers.slice(10, 30), ...sequenceNumbers.slice(20, 30)];
+
+    await pipeline.setCursor(new Timeframe([[feed.key, 9]]));
+    await pipeline.start();
+
+    for await (const block of pipeline.consume()) {
+      processedSequenceNumbers.push(block.seq);
+
+      if (processedSequenceNumbers.length === 20) {
+        // not awaited to avoid a deadlock.
+        void pipeline.pause().then(async () => {
+          await pipeline.setCursor(new Timeframe([[feed.key, 19]]));
+          await pipeline.unpause();
+        });
+      }
+      if (processedSequenceNumbers.length === 30) {
+        void pipeline.stop();
+      }
+    }
+
+    expect(processedSequenceNumbers).toEqual(expectedSequenceNumbers);
+  });
+
+  test('cursor change while polling', async () => {
+    const pipeline = new Pipeline();
+    onTestFinished(() => pipeline.stop());
+
+    const builder = new TestFeedBuilder();
+    const feedStore = builder.createFeedStore();
+    const key = await builder.keyring.createKey();
+    const feed = await feedStore.openFeed(key, { writable: true });
+    await pipeline.addFeed(feed);
+
+    for (const _ of range(20)) {
+      await feed.appendWithReceipt(TEST_MESSAGE);
+    }
+    await feed.clear(0, 10);
+
+    const processedSequenceNumbers: number[] = [];
+    const expectedSequenceNumbers = range(20).slice(10);
+    const processedEvent = new Event();
+    setTimeout(async () => {
+      for await (const block of pipeline.consume()) {
+        processedSequenceNumbers.push(block.seq);
+        processedEvent.emit();
+      }
+    });
+
+    await pipeline.start();
+    await sleep(1000);
+
+    await pipeline.pause();
+    await pipeline.setCursor(new Timeframe([[feed.key, 9]]));
+    await pipeline.unpause();
+
+    await processedEvent.waitForCondition(() => processedSequenceNumbers.length === 10);
+    expect(processedSequenceNumbers).toEqual(expectedSequenceNumbers);
+  });
+});