@dxos/echo-pipeline 0.1.34 → 0.1.36-next.ef27157

package/dist/types/src/pipeline/pipeline.d.ts

@@ -36,6 +36,7 @@ export declare class PipelineState {
     get endTimeframe(): Timeframe;
     get timeframe(): Timeframe;
     get targetTimeframe(): Timeframe;
+    get feeds(): FeedWrapper<any>[];
     waitUntilTimeframe(target: Timeframe): Promise<void>;
     setTargetTimeframe(target: Timeframe): void;
     /**

package/dist/types/src/pipeline/pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../../../src/pipeline/pipeline.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,OAAO,EAAmB,MAAM,eAAe,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC5E,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAI5C,OAAO,EAAwD,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAEzG,oBAAY,4BAA4B,GAAG;IACzC;;OAEG;IACH,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB,CAAC;AAEF;;GAEG;AACH,qBAAa,aAAa;IAatB,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,eAAe;IAZzB,SAAgB,eAAe,yCAA+B;IAE9D;;OAEG;IACH,OAAO,CAAC,gBAAgB,CAAwB;IAEhD,OAAO,CAAC,qBAAqB,CAA4B;gBAI/C,SAAS,EAAE,eAAe,CAAC,GAAG,CAAC,EAC/B,eAAe,EAAE,cAAc;IAGzC;;;OAGG;IACH,IAAI,YAAY,cASf;IAED,IAAI,SAAS,cAEZ;IAED,IAAI,eAAe,cAElB;IAEK,kBAAkB,CAAC,MAAM,EAAE,SAAS;IAI1C,kBAAkB,CAAC,MAAM,EAAE,SAAS;IAIpC;;;;;;OAMG;IACG,+BAA+B,CAAC,EACpC,GAAmB,EACnB,OAAO,EACP,YAAmB,EACpB,GAAE,4BAAiC;CAuCrC;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,aAAa,CAAC;IACrB,MAAM,EAAE,UAAU,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;CACzC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,qBAAa,QAAS,YAAW,gBAAgB;IAsB7C,OAAO,CAAC,QAAQ,CAAC,iBAAiB;IArBpC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA8C;IAG9E,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAG9B;IAGH,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiF;IAGxG,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAwB;IAG3D,OAAO,CAAC,OAAO,CAA8C;IAE7D,OAAO,CAAC,OAAO,CAAS;gBAIL,iBAAiB,GAAE,SAA2B;IAOjE,IAAI,KAAK,kBAER;IAED,IAAI,MAAM,IAAI,UAAU,CAAC,WAAW,CAAC,OAAO,CAAC,CAG5C;IAED,QAAQ,IAAI,WAAW,CAAC,WAAW,CAAC,EAAE;IAIhC,OAAO,CAAC,IAAI,EAAE,WAAW,CAAC,WAAW,CAAC;IAI5C,OAAO,CAAC,OAAO,EAAE,SAAS;IAI1B,YAAY,CAAC,IAAI,EAAE,WAAW,CAAC,WAAW,CAAC;IAarC,KAAK;IAML,IAAI;IAOV;;;OAGG;IACI,OAAO,IAAI,aAAa,CAAC,gBAAgB,CAAC;CAelD"}
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../../../src/pipeline/pipeline.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,OAAO,EAAmB,MAAM,eAAe,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC5E,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAI5C,OAAO,EAAyC,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAE1F,oBAAY,4BAA4B,GAAG;IACzC;;OAEG;IACH,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB,CAAC;AAEF;;GAEG;AACH,qBAAa,aAAa;IAatB,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,eAAe;IAZzB,SAAgB,eAAe,yCAA+B;IAE9D;;OAEG;IACH,OAAO,CAAC,gBAAgB,CAAwB;IAEhD,OAAO,CAAC,qBAAqB,CAA4B;gBAI/C,SAAS,EAAE,eAAe,CAAC,GAAG,CAAC,EAC/B,eAAe,EAAE,cAAc;IAGzC;;;OAGG;IAEH,IAAI,YAAY,cASf;IAED,IAAI,SAAS,cAEZ;IAED,IAAI,eAAe,cAElB;IAED,IAAI,KAAK,uBAER;IAEK,kBAAkB,CAAC,MAAM,EAAE,SAAS;IAI1C,kBAAkB,CAAC,MAAM,EAAE,SAAS;IAIpC;;;;;;OAMG;IACG,+BAA+B,CAAC,EACpC,GAAmB,EACnB,OAAO,EACP,YAAmB,EACpB,GAAE,4BAAiC;CAuCrC;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,aAAa,CAAC;IACrB,MAAM,EAAE,UAAU,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;CACzC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,qBAAa,QAAS,YAAW,gBAAgB;IAsB7C,OAAO,CAAC,QAAQ,CAAC,iBAAiB;IArBpC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA8C;IAG9E,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAG9B;IAGH,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiF;IAGxG,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAwB;IAG3D,OAAO,CAAC,OAAO,CAA8C;IAE7D,OAAO,CAAC,OAAO,CAAS;gBAIL,iBAAiB,GAAE,SAA2B;IAOjE,IAAI,KAAK,kBAER;IAED,IAAI,MAAM,IAAI,UAAU,CAAC,WAAW,CAAC,OAAO,CAAC,CAG5C;IAED,QAAQ,IAAI,WAAW,CAAC,WAAW,CAAC,EAAE;IAMhC,OAAO,CAAC,IAAI,EAAE,WAAW,CAAC,WAAW,CAAC;IAI5C,OAAO,CAAC,OAAO,EAAE,SAAS;IAI1B,YAAY,CAAC,IAAI,EAAE,WAAW,CAAC,WAAW,CAAC;IAcrC,KAAK;IAOL,IAAI;IAOV;;;OAGG;IACI,OAAO,IAAI,aAAa,CAAC,gBAAgB,CAAC;CAelD"}

package/dist/types/src/pipeline/timeframe-clock.d.ts

@@ -4,6 +4,7 @@ import { PublicKey } from '@dxos/keys';
 import { Timeframe } from '@dxos/timeframe';
 export declare const mapTimeframeToFeedIndexes: (timeframe: Timeframe) => FeedIndex[];
 export declare const mapFeedIndexesToTimeframe: (indexes: FeedIndex[]) => Timeframe;
+export declare const startAfter: (timeframe: Timeframe) => FeedIndex[];
 /**
  * Keeps state of the last timeframe that was processed by ECHO.
  */

package/dist/types/src/pipeline/timeframe-clock.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"timeframe-clock.d.ts","sourceRoot":"","sources":["../../../../src/pipeline/timeframe-clock.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAEpC,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,eAAO,MAAM,yBAAyB,cAAe,SAAS,KAAG,SAAS,EACN,CAAC;AAErE,eAAO,MAAM,yBAAyB,YAAa,SAAS,EAAE,KAAG,SACK,CAAC;AAEvE;;GAEG;AACH,qBAAa,cAAc;IAKvB,OAAO,CAAC,UAAU;IAJpB,QAAQ,CAAC,MAAM,mBAA0B;gBAI/B,UAAU,YAAkB;IAGtC,IAAI,SAAS,cAEZ;IAED,eAAe,CAAC,GAAG,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM;IAK3C,OAAO,CAAC,SAAS,EAAE,SAAS;IAMtB,gBAAgB,CAAC,MAAM,EAAE,SAAS;CAYzC"}
+{"version":3,"file":"timeframe-clock.d.ts","sourceRoot":"","sources":["../../../../src/pipeline/timeframe-clock.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAEpC,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,eAAO,MAAM,yBAAyB,cAAe,SAAS,KAAG,SAAS,EACN,CAAC;AAErE,eAAO,MAAM,yBAAyB,YAAa,SAAS,EAAE,KAAG,SACK,CAAC;AAEvE,eAAO,MAAM,UAAU,cAAe,SAAS,KAAG,SAAS,EACoB,CAAC;AAEhF;;GAEG;AACH,qBAAa,cAAc;IAKvB,OAAO,CAAC,UAAU;IAJpB,QAAQ,CAAC,MAAM,mBAA0B;gBAI/B,UAAU,YAAkB;IAGtC,IAAI,SAAS,cAEZ;IAED,eAAe,CAAC,GAAG,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM;IAK3C,OAAO,CAAC,SAAS,EAAE,SAAS;IAMtB,gBAAgB,CAAC,MAAM,EAAE,SAAS;CAYzC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dxos/echo-pipeline",
-  "version": "0.1.34",
+  "version": "0.1.36-next.ef27157",
   "description": "ECHO database.",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
@@ -20,40 +20,41 @@
   ],
   "dependencies": {
     "debug": "^4.3.3",
-    "@dxos/async": "0.1.34",
-    "@dxos/codec-protobuf": "0.1.34",
-    "@dxos/context": "0.1.34",
-    "@dxos/credentials": "0.1.34",
-    "@dxos/crypto": "0.1.34",
-    "@dxos/debug": "0.1.34",
-    "@dxos/document-model": "0.1.34",
-    "@dxos/echo-db": "0.1.34",
-    "@dxos/feed-store": "0.1.34",
-    "@dxos/hypercore": "0.1.34",
-    "@dxos/keyring": "0.1.34",
-    "@dxos/keys": "0.1.34",
-    "@dxos/log": "0.1.34",
-    "@dxos/messaging": "0.1.34",
-    "@dxos/model-factory": "0.1.34",
-    "@dxos/network-manager": "0.1.34",
-    "@dxos/node-std": "0.1.34",
-    "@dxos/protocols": "0.1.34",
-    "@dxos/random-access-storage": "0.1.34",
-    "@dxos/rpc": "0.1.34",
-    "@dxos/teleport": "0.1.34",
-    "@dxos/teleport-extension-gossip": "0.1.34",
-    "@dxos/teleport-extension-object-sync": "0.1.34",
-    "@dxos/teleport-extension-replicator": "0.1.34",
-    "@dxos/timeframe": "0.1.34",
-    "@dxos/typings": "0.1.34",
-    "@dxos/util": "0.1.34"
+    "@dxos/async": "0.1.36-next.ef27157",
+    "@dxos/codec-protobuf": "0.1.36-next.ef27157",
+    "@dxos/context": "0.1.36-next.ef27157",
+    "@dxos/credentials": "0.1.36-next.ef27157",
+    "@dxos/crypto": "0.1.36-next.ef27157",
+    "@dxos/debug": "0.1.36-next.ef27157",
+    "@dxos/document-model": "0.1.36-next.ef27157",
+    "@dxos/echo-db": "0.1.36-next.ef27157",
+    "@dxos/feed-store": "0.1.36-next.ef27157",
+    "@dxos/hypercore": "0.1.36-next.ef27157",
+    "@dxos/keyring": "0.1.36-next.ef27157",
+    "@dxos/keys": "0.1.36-next.ef27157",
+    "@dxos/log": "0.1.36-next.ef27157",
+    "@dxos/messaging": "0.1.36-next.ef27157",
+    "@dxos/model-factory": "0.1.36-next.ef27157",
+    "@dxos/network-manager": "0.1.36-next.ef27157",
+    "@dxos/node-std": "0.1.36-next.ef27157",
+    "@dxos/protocols": "0.1.36-next.ef27157",
+    "@dxos/random-access-storage": "0.1.36-next.ef27157",
+    "@dxos/rpc": "0.1.36-next.ef27157",
+    "@dxos/teleport": "0.1.36-next.ef27157",
+    "@dxos/teleport-extension-gossip": "0.1.36-next.ef27157",
+    "@dxos/teleport-extension-object-sync": "0.1.36-next.ef27157",
+    "@dxos/teleport-extension-replicator": "0.1.36-next.ef27157",
+    "@dxos/timeframe": "0.1.36-next.ef27157",
+    "@dxos/typings": "0.1.36-next.ef27157",
+    "@dxos/util": "0.1.36-next.ef27157"
   },
   "devDependencies": {
     "@types/debug": "^4.1.7",
+    "fast-check": "~3.3.0",
     "hypercore-protocol": "^8.0.7",
     "source-map-support": "^0.5.12",
     "wait-for-expect": "^3.0.2",
-    "@dxos/text-model": "0.1.34"
+    "@dxos/text-model": "0.1.36-next.ef27157"
   },
   "publishConfig": {
     "access": "public"

package/src/pipeline/pipeline.test.ts

@@ -2,7 +2,16 @@
 // Copyright 2022 DXOS.org
 //
 
+import expect from 'expect';
+import * as fc from 'fast-check';
+import { inspect } from 'util';
+
+import { asyncTimeout } from '@dxos/async';
 import { checkType } from '@dxos/debug';
+import { FeedStore, FeedWrapper } from '@dxos/feed-store';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { FeedMessageBlock } from '@dxos/protocols';
 import { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { describe, test, afterTest } from '@dxos/test';
 import { Timeframe } from '@dxos/timeframe';
@@ -76,4 +85,209 @@ describe('pipeline/Pipeline', () => {
       }
     }
   });
+
+  test
+    .skip('stress', async () => {
+      const builder = new TestFeedBuilder();
+
+      const NUM_AGENTS = 2;
+      const NUM_MESSAGES = 100;
+
+      class Agent {
+        public startingTimeframe = new Timeframe();
+        public pipeline!: Pipeline;
+        public feed!: FeedWrapper<FeedMessage>;
+        public messages: FeedMessageBlock[] = [];
+        public writePromise: Promise<any> = Promise.resolve();
+
+        constructor(public id: string, public feedStore: FeedStore<FeedMessage>) {}
+
+        async open() {
+          const key = await builder.keyring.createKey();
+          this.feed = await this.feedStore.openFeed(key, { writable: true });
+        }
+
+        async start() {
+          this.pipeline = new Pipeline(this.startingTimeframe);
+
+          await this.pipeline.start();
+
+          // NOTE: not awaiting here breaks the test.
+          await Promise.all(this.feedStore.feeds.map((feed) => this.pipeline.addFeed(feed)));
+          this.pipeline.setWriteFeed(this.feed);
+
+          // consume in async task.
+          void this.consume();
+        }
+
+        async stop() {
+          await this.pipeline.stop();
+          this.startingTimeframe = this.pipeline.state.timeframe;
+        }
+
+        async close() {
+          await this.feed.close();
+        }
+
+        write(message: FeedMessage.Payload) {
+          const prev = this.writePromise;
+          const promise = this.pipeline.writer!.write(message);
+          this.writePromise = Promise.all([prev, promise]);
+        }
+
+        async consume() {
+          for await (const msg of this.pipeline.consume()) {
+            this.messages.push(msg);
+          }
+          log('stopped consuming');
+        }
+      }
+
+      type Model = {};
+      type Real = {
+        feedStore: FeedStore<FeedMessage>;
+        agents: Map<string, Agent>;
+      };
+
+      class WriteCommand implements fc.AsyncCommand<Model, Real> {
+        constructor(public agent: string, public count: number) {}
+
+        check = () => true;
+
+        async run(model: Model, real: Real) {
+          console.log(`WriteCommand(${this.agent}, ${this.count})`);
+
+          const agent = real.agents.get(this.agent)!;
+
+          const toWrite = Math.min(this.count, NUM_MESSAGES - agent.feed.length);
+          if (toWrite > 0) {
+            for (const _ of range(toWrite)) {
+              agent.write({}); // Content is not important.
+            }
+          }
+        }
+
+        toString = () => `WriteCommand(${this.agent}, ${this.count})`;
+      }
+
+      class SyncCommand implements fc.AsyncCommand<Model, Real> {
+        check = () => true;
+
+        async run(model: Model, real: Real) {
+          console.log('SyncCommand()');
+          const targets: any = {};
+
+          try {
+            for (const agent of real.agents.values()) {
+              await agent.writePromise;
+            }
+
+            for (const agent of real.agents.values()) {
+              targets[agent.id] = agent.pipeline.state.endTimeframe;
+              if (agent.pipeline.state.endTimeframe.isEmpty()) {
+                console.log('empty endtimeframe', {
+                  id: agent.id,
+                  endTimeframe: agent.pipeline.state.endTimeframe,
+                  feeds: agent.pipeline.getFeeds().map((feed) => [feed.key.toString(), feed.length])
+                });
+              }
+              await asyncTimeout(agent.pipeline.state.waitUntilTimeframe(agent.pipeline.state.endTimeframe), 1000);
+            }
+
+            const tf: Timeframe = real.agents.values().next().value!.pipeline.state.timeframe;
+            for (const agent of real.agents.values()) {
+              expect(agent.pipeline.state.timeframe.equals(tf)).toEqual(true);
+            }
+
+            const totalMessages = real.feedStore.feeds.reduce((acc, feed) => acc + feed.length, 0);
+            for (const agent of real.agents.values()) {
+              expect(agent.messages.length).toEqual(totalMessages);
+            }
+          } catch (err) {
+            console.log(
+              inspect(
+                {
+                  agents: Array.from(real.agents.values()).map((agent) => ({
+                    id: agent.id,
+                    messages: agent.messages.length,
+                    feeds: agent.pipeline.getFeeds().map((feed) => [feed.key, feed.length]),
+                    timeframe: agent.pipeline.state.timeframe,
+                    endTimeframe: agent.pipeline.state.endTimeframe
+                  })),
+                  feeds: real.feedStore.feeds.map((feed) => [feed.key, feed.length]),
+                  targets
+                },
+                false,
+                null,
+                true
+              )
+            );
+            throw err;
+          }
+        }
+
+        toString = () => 'SyncCommand()';
+      }
+
+      class RestartCommand implements fc.AsyncCommand<Model, Real> {
+        constructor(public agent: string) {}
+
+        check = () => true;
+
+        async run(model: Model, real: Real) {
+          console.log(`RestartCommand(${this.agent})`);
+
+          const agent = real.agents.get(this.agent)!;
+
+          await agent.writePromise;
+          await agent.stop();
+          await agent.start();
+        }
+
+        toString = () => `RestartCommand(${this.agent})`;
+      }
+
+      const agentIds = range(NUM_AGENTS).map(() => PublicKey.random().toHex().slice(0, 8));
+      const anAgentId = fc.constantFrom(...agentIds);
+
+      const commands = fc.commands(
+        [
+          fc.tuple(anAgentId, fc.integer({ min: 1, max: 10 })).map(([agent, count]) => new WriteCommand(agent, count)),
+          fc.constant(new SyncCommand()),
+          anAgentId.map((agent) => new RestartCommand(agent))
+        ],
+        { size: 'large' }
+      );
+
+      const model = fc.asyncProperty(commands, async (commands) => {
+        const feedStore = builder.createFeedStore();
+
+        const agents = new Map(agentIds.map((id) => [id, new Agent(id, feedStore)]));
+        await Promise.all(Array.from(agents.values()).map((agent) => agent.open()));
+        await Promise.all(Array.from(agents.values()).map((agent) => agent.start()));
+
+        const setup: fc.ModelRunSetup<Model, Real> = () => ({
+          model: {},
+          real: {
+            feedStore,
+            agents
+          }
+        });
+
+        try {
+          await fc.asyncModelRun(setup, [...commands, new SyncCommand()]);
+        } finally {
+          await Promise.all(Array.from(agents.values()).map((agent) => agent.stop()));
+          await Promise.all(Array.from(agents.values()).map((agent) => agent.close()));
+        }
+      });
+
+      const examples: [commands: Iterable<fc.AsyncCommand<Model, Real, boolean>>][] = [
+        [[new WriteCommand(agentIds[0], 10), new WriteCommand(agentIds[1], 10), new SyncCommand()]],
+        [[new WriteCommand(agentIds[0], 4), new RestartCommand(agentIds[0]), new SyncCommand()]]
+      ];
+
+      await fc.assert(model, { examples });
+    })
+    .timeout(60_000);
 });

package/src/pipeline/pipeline.ts

@@ -4,7 +4,7 @@
 
 import assert from 'node:assert';
 
-import { sleep, Trigger } from '@dxos/async';
+import { sleep, synchronized, Trigger } from '@dxos/async';
 import { Context, rejectOnDispose } from '@dxos/context';
 import { FeedSetIterator, FeedWrapper, FeedWriter } from '@dxos/feed-store';
 import { PublicKey } from '@dxos/keys';
@@ -15,7 +15,7 @@ import { Timeframe } from '@dxos/timeframe';
 
 import { createMappedFeedWriter } from '../common';
 import { createMessageSelector } from './message-selector';
-import { mapFeedIndexesToTimeframe, mapTimeframeToFeedIndexes, TimeframeClock } from './timeframe-clock';
+import { mapFeedIndexesToTimeframe, startAfter, TimeframeClock } from './timeframe-clock';
 
 export type WaitUntilReachedTargetParams = {
   /**
@@ -54,13 +54,14 @@ export class PipelineState {
    * Latest theoretical timeframe based on the last mutation in each feed.
    * NOTE: This might never be reached if the mutation dependencies
    */
+  // TODO(dmaretskyi): Rename `totalTimeframe`? or `lastTimeframe`.
   get endTimeframe() {
     return mapFeedIndexesToTimeframe(
       this._iterator.feeds
-        .filter((feed) => feed.properties.length > 0)
+        .filter((feed) => feed.length > 0)
         .map((feed) => ({
           feedKey: feed.key,
-          index: feed.properties.length - 1
+          index: feed.length - 1
         }))
     );
   }
@@ -73,6 +74,10 @@ export class PipelineState {
     return this._targetTimeframe ? this._targetTimeframe : new Timeframe();
   }
 
+  get feeds() {
+    return this._iterator.feeds;
+  }
+
   async waitUntilTimeframe(target: Timeframe) {
     await this._timeframeClock.waitUntilReached(target);
   }
@@ -175,7 +180,7 @@ export class Pipeline implements PipelineAccessor {
 
   // Inbound feed stream.
   private readonly _feedSetIterator = new FeedSetIterator<FeedMessage>(createMessageSelector(this._timeframeClock), {
-    start: mapTimeframeToFeedIndexes(this._initialTimeframe),
+    start: startAfter(this._initialTimeframe),
     stallTimeout: 1000
   });
 
@@ -212,6 +217,8 @@ export class Pipeline implements PipelineAccessor {
     return this._feedSetIterator.feeds;
   }
 
+  // NOTE: This cannot be synchronized with `stop` because stop waits for the mutation processing to complete,
+  // which might be opening feeds during the mutation processing, which w
   async addFeed(feed: FeedWrapper<FeedMessage>) {
     await this._feedSetIterator.addFeed(feed);
   }
@@ -233,12 +240,14 @@ export class Pipeline implements PipelineAccessor {
     );
   }
 
+  @synchronized
   async start() {
     log('starting...', {});
     await this._feedSetIterator.open();
     log('started');
   }
 
+  @synchronized
   async stop() {
     log('stopping...', {});
     await this._feedSetIterator.close();

package/src/pipeline/timeframe-clock.ts

@@ -15,6 +15,9 @@ export const mapTimeframeToFeedIndexes = (timeframe: Timeframe): FeedIndex[] =>
 export const mapFeedIndexesToTimeframe = (indexes: FeedIndex[]): Timeframe =>
   new Timeframe(indexes.map(({ feedKey, index }) => [feedKey, index]));
 
+export const startAfter = (timeframe: Timeframe): FeedIndex[] =>
+  timeframe.frames().map(([feedKey, index]) => ({ feedKey, index: index + 1 }));
+
 /**
  * Keeps state of the last timeframe that was processed by ECHO.
  */
@@ -42,7 +45,7 @@ export class TimeframeClock {
 
   @timed(5_000)
   async waitUntilReached(target: Timeframe) {
-    log.debug('waitUntilReached', { target, current: this._timeframe });
+    log.info('waitUntilReached', { target, current: this._timeframe });
     await this.update.waitForCondition(() => {
       log('check if reached', {
         target,

package/src/space/data-pipeline.ts

@@ -53,7 +53,7 @@ const TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
 /**
  * Flag to disable automatic local snapshots.
  */
-const DISABLE_SNAPSHOT_CACHE = false;
+const DISABLE_SNAPSHOT_CACHE = true;
 
 /**
  * Controls data pipeline in the space.
@@ -106,7 +106,7 @@ export class DataPipeline {
     }
 
     this._spaceContext = spaceContext;
-    if (this._params.snapshotId) {
+    if (this._params.snapshotId && !DISABLE_SNAPSHOT_CACHE) {
       this._snapshot = await this._params.snapshotManager.load(this._params.snapshotId);
       this._lastAutomaticSnapshotTimeframe = this._snapshot?.timeframe ?? new Timeframe();
     }