@dxos/echo-pipeline 0.1.41 → 0.1.45-next.45667d4

package/dist/types/src/testing/test-agent-builder.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"test-agent-builder.d.ts","sourceRoot":"","sources":["../../../../src/testing/test-agent-builder.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAuB,0BAA0B,EAA0B,MAAM,iBAAiB,CAAC;AAE1G,OAAO,EAAwD,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAC7G,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAiB,OAAO,EAAe,MAAM,6BAA6B,CAAC;AAClF,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAKnE,OAAO,EAA0C,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACtG,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAEtD,oBAAY,sBAAsB,GAAG,MAAM,cAAc,CAAC;AAE1D,eAAO,MAAM,4BAA4B,kBACvB,0BAA0B,KAAG,sBAKzC,CAAC;AAEP,eAAO,MAAM,+BAA+B,cAC9B,MAAM,KAAG,sBAKjB,CAAC;AAEP,oBAAY,uBAAuB,GAAG;IACpC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,sBAAsB,CAAC,EAAE,sBAAsB,CAAC;CACjD,CAAC;AAEF;;GAEG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwD;IAChF,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAU;IACnC,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAyB;gBAErD,EAAE,OAAO,EAAE,sBAAsB,EAAE,GAAE,uBAA4B;IAMvE,KAAK;IAIX,IAAI,MAAM,gBAET;IAED,QAAQ,CAAC,SAAS,EAAE,SAAS;IAIvB,UAAU,IAAI,OAAO,CAAC,SAAS,CAAC;CAYvC;AAED;;GAEG;AACH,qBAAa,SAAS;IAOlB,OAAO,CAAC,QAAQ,CAAC,uBAAuB;IACxC,OAAO,CAAC,QAAQ,CAAC,YAAY;aACb,WAAW,EAAE,SAAS;aACtB,SAAS,EAAE,SAAS;IATtC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoD;IAE5E,SAAgB,OAAO,EAAE,OAAO,CAAC;IACjC,SAAgB,SAAS,EAAE,SAAS,CAAC,WAAW,CAAC,CAAC;gBAG/B,uBAAuB,EAAE,sBAAsB,EAC/C,YAAY,EAAE,eAAe,EAC9B,WAAW,EAAE,SAAS,EACtB,SAAS,EAAE,SAAS;IAMhC,KAAK;IAIX,IAAI,MAAM,YAET;IAED,QAAQ,CAAC,QAAQ,EAAE,SAAS;IAI5B,kBAAkB;IAOZ,WAAW,CACf,WAAW,GAAE,SAA4B,EACzC,QAAQ,CAAC,EAAE,SAAS,EACpB,UAAU,CAAC,EAAE,SAAS,GACrB,OAAO,CAAC,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;IA6CjC,mBAAmB,CAAC,KAAK,EAAE,SAAS,EAAE,MAAM,CAAC,EAAE,MAAM;IAkBrD,YAAY;IAMZ,cAAc,CAAC,MAAM,CAAC,EAAE,MAAM;CAQ/B"}
+{"version":3,"file":"test-agent-builder.d.ts","sourceRoot":"","sources":["../../../../src/testing/test-agent-builder.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAuB,0BAA0B,EAA0B,MAAM,iBAAiB,CAAC;AAC1G,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAwD,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAC7G,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sCAAsC,CAAC;AAGxE,OAAO,EAAiB,OAAO,EAAe,MAAM,6BAA6B,CAAC;AAClF,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAGnE,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAA0C,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACtG,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAEtD,MAAM,MAAM,sBAAsB,GAAG,MAAM,cAAc,CAAC;AAE1D,eAAO,MAAM,4BAA4B,kBACvB,0BAA0B,KAAG,sBAKzC,CAAC;AAEP,eAAO,MAAM,+BAA+B,cAC9B,MAAM,KAAG,sBAKjB,CAAC;AAEP,MAAM,MAAM,uBAAuB,GAAG;IACpC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,sBAAsB,CAAC,EAAE,sBAAsB,CAAC;CACjD,CAAC;AAEF;;GAEG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwD;IAChF,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAU;IACnC,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAyB;gBAErD,EAAE,OAAO,EAAE,sBAAsB,EAAE,GAAE,uBAA4B;IAMvE,KAAK;IAIX,IAAI,MAAM,gBAET;IAED,QAAQ,CAAC,SAAS,EAAE,SAAS;IAIvB,UAAU,IAAI,OAAO,CAAC,SAAS,CAAC;CAYvC;AAED;;GAEG;AACH,qBAAa,SAAS;IAoBlB,OAAO,CAAC,QAAQ,CAAC,uBAAuB;IACxC,OAAO,CAAC,QAAQ,CAAC,YAAY;aACb,WAAW,EAAE,SAAS;aACtB,SAAS,EAAE,SAAS;IAtBtC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoD;IAE5E,SAAgB,OAAO,EAAE,OAAO,CAAC;IACjC,SAAgB,OAAO,EAAE,OAAO,CAAC;IACjC,SAAgB,SAAS,EAAE,SAAS,CAAC,WAAW,CAAC,CAAC;IAElD,OAAO,CAAC,cAAc,CAAC,CAAgB;IACvC,IAAI,aAAa,kBAEhB;IAED,OAAO,CAAC,cAAc,CAAC,CAAgB;IACvC,IAAI,aAAa,kBAEhB;IAEM,YAAY,eAAmD;gBAGnD,uBAAuB,EAAE,sBAAsB,EAC/C,YAAY,EAAE,eAAe,EAC9B,WAAW,EAAE,SAAS,EACtB,SAAS,EAAE,SAAS;IAOhC,KAAK;IAIX,IAAI,MAAM,YAET;IAED,QAAQ,CAAC,QAAQ,EAAE,SAAS;IAI5B,OAAO,CAAC,aAAa,CAAC,CAAe;IACrC,IAAI,YAAY,iBAQf;IAEK,WAAW,CACf,WAAW,GAAE,SAA4B,EACzC,QAAQ,CAAC,EAAE,SAAS,EACpB,UAAU,CAAC,EAAE,SAAS,EACtB,OAAO,CAAC,EAAE,SAAS,GAClB,OAAO,CAAC,KAAK,CAAC;IAgDjB,mBAAmB,CAAC,KAAK,EAAE,SAAS,EAAE,MAAM,CAAC,EAAE,MAAM;IAkBrD,YAAY;IAMZ,cAAc,CAAC,MAAM,CAAC,EAAE,MAAM;IASxB,YAAY,CAAC,KAAK,EAAE,KAAK;CAchC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dxos/echo-pipeline",
-  "version": "0.1.41",
+  "version": "0.1.45-next.45667d4",
   "description": "ECHO database.",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
@@ -21,34 +21,34 @@
   "dependencies": {
     "crc-32": "^1.2.2",
     "debug": "^4.3.3",
-    "@dxos/async": "0.1.41",
-    "@dxos/codec-protobuf": "0.1.41",
-    "@dxos/context": "0.1.41",
-    "@dxos/credentials": "0.1.41",
-    "@dxos/crypto": "0.1.41",
-    "@dxos/debug": "0.1.41",
-    "@dxos/document-model": "0.1.41",
-    "@dxos/echo-db": "0.1.41",
-    "@dxos/errors": "0.1.41",
-    "@dxos/feed-store": "0.1.41",
-    "@dxos/hypercore": "0.1.41",
-    "@dxos/keyring": "0.1.41",
-    "@dxos/keys": "0.1.41",
-    "@dxos/log": "0.1.41",
-    "@dxos/messaging": "0.1.41",
-    "@dxos/model-factory": "0.1.41",
-    "@dxos/network-manager": "0.1.41",
-    "@dxos/node-std": "0.1.41",
-    "@dxos/protocols": "0.1.41",
-    "@dxos/random-access-storage": "0.1.41",
-    "@dxos/rpc": "0.1.41",
-    "@dxos/teleport": "0.1.41",
-    "@dxos/teleport-extension-gossip": "0.1.41",
-    "@dxos/teleport-extension-object-sync": "0.1.41",
-    "@dxos/teleport-extension-replicator": "0.1.41",
-    "@dxos/timeframe": "0.1.41",
-    "@dxos/typings": "0.1.41",
-    "@dxos/util": "0.1.41"
+    "@dxos/async": "0.1.45-next.45667d4",
+    "@dxos/codec-protobuf": "0.1.45-next.45667d4",
+    "@dxos/context": "0.1.45-next.45667d4",
+    "@dxos/credentials": "0.1.45-next.45667d4",
+    "@dxos/crypto": "0.1.45-next.45667d4",
+    "@dxos/debug": "0.1.45-next.45667d4",
+    "@dxos/document-model": "0.1.45-next.45667d4",
+    "@dxos/echo-db": "0.1.45-next.45667d4",
+    "@dxos/errors": "0.1.45-next.45667d4",
+    "@dxos/feed-store": "0.1.45-next.45667d4",
+    "@dxos/hypercore": "0.1.45-next.45667d4",
+    "@dxos/keyring": "0.1.45-next.45667d4",
+    "@dxos/keys": "0.1.45-next.45667d4",
+    "@dxos/log": "0.1.45-next.45667d4",
+    "@dxos/messaging": "0.1.45-next.45667d4",
+    "@dxos/model-factory": "0.1.45-next.45667d4",
+    "@dxos/network-manager": "0.1.45-next.45667d4",
+    "@dxos/node-std": "0.1.45-next.45667d4",
+    "@dxos/protocols": "0.1.45-next.45667d4",
+    "@dxos/random-access-storage": "0.1.45-next.45667d4",
+    "@dxos/rpc": "0.1.45-next.45667d4",
+    "@dxos/teleport": "0.1.45-next.45667d4",
+    "@dxos/teleport-extension-gossip": "0.1.45-next.45667d4",
+    "@dxos/teleport-extension-object-sync": "0.1.45-next.45667d4",
+    "@dxos/teleport-extension-replicator": "0.1.45-next.45667d4",
+    "@dxos/timeframe": "0.1.45-next.45667d4",
+    "@dxos/typings": "0.1.45-next.45667d4",
+    "@dxos/util": "0.1.45-next.45667d4"
   },
   "devDependencies": {
     "@types/debug": "^4.1.7",
@@ -56,7 +56,7 @@
     "hypercore-protocol": "^8.0.7",
     "source-map-support": "^0.5.12",
     "wait-for-expect": "^3.0.2",
-    "@dxos/text-model": "0.1.41"
+    "@dxos/text-model": "0.1.45-next.45667d4"
   },
   "publishConfig": {
     "access": "public"

package/src/common/feeds.ts

@@ -12,12 +12,12 @@ import { MaybePromise } from '@dxos/util';
  */
 export const createMappedFeedWriter = <Source extends {}, Target extends {}>(
   mapper: (arg: Source) => MaybePromise<Target>,
-  writer: FeedWriter<Target>
+  writer: FeedWriter<Target>,
 ): FeedWriter<Source> => {
   assert(mapper);
   assert(writer);
 
   return {
-    write: async (data: Source, options) => await writer.write(await mapper(data), options)
+    write: async (data: Source, options) => await writer.write(await mapper(data), options),
   };
 };

package/src/dbhost/data-service-host.ts

@@ -20,13 +20,13 @@ import { ComplexMap } from '@dxos/util';
 // TODO(burdon): Move to client-services.
 export class DataServiceHost {
   private readonly _clientTagMap = new ComplexMap<[feedKey: PublicKey, seq: number], string>(
-    ([feedKey, seq]) => `${feedKey.toHex()}:${seq}`
+    ([feedKey, seq]) => `${feedKey.toHex()}:${seq}`,
   );
 
   constructor(
     private readonly _itemManager: ItemManager,
     private readonly _itemDemuxer: ItemDemuxer,
-    private readonly _writeStream?: FeedWriter<DataMessage>
+    private readonly _writeStream?: FeedWriter<DataMessage>,
   ) {}
 
   /**
@@ -38,8 +38,8 @@ export class DataServiceHost {
       const objects = Array.from(this._itemManager.entities.values()).map((entity) => entity.createSnapshot());
       next({
         batch: {
-          objects
-        }
+          objects,
+        },
       });
 
       // subscribe to mutations
@@ -65,7 +65,7 @@ export class DataServiceHost {
           clientTag,
           feedKey: message.meta.feedKey,
           seq: message.meta.seq,
-          batch
+          batch,
         });
       });
     });
@@ -83,11 +83,11 @@ export class DataServiceHost {
           ...object,
           mutations: object.mutations?.map((mutation) => ({
             ...mutation,
-            meta: undefined
+            meta: undefined,
           })),
-          meta: undefined
-        }))
-      }
+          meta: undefined,
+        })),
+      },
     };
     const receipt = await this._writeStream.write(message, {
       afterWrite: async (receipt) => {
@@ -96,7 +96,7 @@ export class DataServiceHost {
           log('tag', { clientTag: request.clientTag, feedKey: receipt.feedKey, seq: receipt.seq });
           this._clientTagMap.set([receipt.feedKey, receipt.seq], request.clientTag);
         }
-      }
+      },
     });
 
     return receipt;

package/src/dbhost/data-service.ts

@@ -13,7 +13,7 @@ import {
   MutationReceipt,
   SubscribeRequest,
   EchoEvent,
-  WriteRequest
+  WriteRequest,
 } from '@dxos/protocols/proto/dxos/echo/service';
 import { ComplexMap } from '@dxos/util';
 

package/src/dbhost/database-host.ts

@@ -8,7 +8,6 @@ import { EchoProcessor, ItemDemuxer, ItemManager } from '@dxos/echo-db';
 import { FeedWriter } from '@dxos/feed-store';
 import { ModelFactory } from '@dxos/model-factory';
 import { DataMessage } from '@dxos/protocols/proto/dxos/echo/feed';
-import { EchoSnapshot } from '@dxos/protocols/proto/dxos/echo/snapshot';
 
 import { DataServiceHost } from './data-service-host';
 
@@ -22,10 +21,7 @@ export class DatabaseHost {
   private _itemManager!: ItemManager;
   public _itemDemuxer!: ItemDemuxer;
 
-  constructor(
-    private readonly _outboundStream: FeedWriter<DataMessage> | undefined,
-    private readonly _snapshot?: EchoSnapshot
-  ) {}
+  constructor(private readonly _outboundStream: FeedWriter<DataMessage> | undefined) {}
 
   get isReadOnly(): boolean {
     return !!this._outboundStream;
@@ -41,10 +37,6 @@ export class DatabaseHost {
 
     this._itemDemuxer = new ItemDemuxer(itemManager, modelFactory);
     this._echoProcessor = this._itemDemuxer.open();
-
-    if (this._snapshot) {
-      this._itemDemuxer.restoreFromSnapshot(this._snapshot);
-    }
   }
 
   async close() {}

package/src/dbhost/snapshot-store.test.ts

@@ -20,8 +20,8 @@ describe('SnapshotStore', () => {
       spaceKey: PublicKey.random().asBuffer(),
       timeframe: new Timeframe(),
       database: {
-        items: []
-      }
+        items: [],
+      },
     };
 
     const key = await store.saveSnapshot(snapshot);

package/src/dbhost/snapshot-store.ts

@@ -4,6 +4,7 @@
 
 import { subtleCrypto } from '@dxos/crypto';
 import { schema } from '@dxos/protocols';
+import { StoredSnapshotInfo } from '@dxos/protocols/proto/dxos/devtools/host';
 import { SpaceSnapshot } from '@dxos/protocols/proto/dxos/echo/snapshot';
 import { Directory } from '@dxos/random-access-storage';
 
@@ -11,7 +12,7 @@ export class SnapshotStore {
   // prettier-ignore
   constructor(
     private readonly _directory: Directory
-  ) {}
+  ) { }
 
   async saveSnapshot(snapshot: SpaceSnapshot): Promise<string> {
     const encoded = schema.getCodecForType('dxos.echo.snapshot.SpaceSnapshot').encode(snapshot);
@@ -42,4 +43,18 @@ export class SnapshotStore {
       await file.close();
     }
   }
+
+  async listSnapshots(): Promise<StoredSnapshotInfo[]> {
+    const entries = await this._directory.list();
+
+    return await Promise.all(
+      entries.map(async (key) => {
+        const { size } = await this._directory.getOrCreateFile(key).stat();
+        return {
+          key,
+          size,
+        };
+      }),
+    );
+  }
 }

package/src/metadata/metadata-store.ts

@@ -10,7 +10,7 @@ import { DataCorruptionError } from '@dxos/errors';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 import { schema } from '@dxos/protocols';
-import { EchoMetadata, SpaceMetadata, IdentityRecord } from '@dxos/protocols/proto/dxos/echo/metadata';
+import { EchoMetadata, SpaceMetadata, IdentityRecord, SpaceCache } from '@dxos/protocols/proto/dxos/echo/metadata';
 import { Directory } from '@dxos/random-access-storage';
 import { Timeframe } from '@dxos/timeframe';
 
@@ -30,7 +30,7 @@ const emptyEchoMetadata = (): EchoMetadata => ({
   version: STORAGE_VERSION,
   spaces: [],
   created: new Date(),
-  updated: new Date()
+  updated: new Date(),
 });
 
 export class MetadataStore {
@@ -95,7 +95,7 @@ export class MetadataStore {
       ...this._metadata,
       version: STORAGE_VERSION,
       created: this._metadata.created ?? new Date(),
-      updated: new Date()
+      updated: new Date(),
     };
 
     const file = this._directory.getOrCreateFile('EchoMetadata');
@@ -119,6 +119,12 @@ export class MetadataStore {
     }
   }
 
+  _getSpace(spaceKey: PublicKey): SpaceMetadata {
+    const space = this.spaces.find((space) => space.key === spaceKey);
+    assert(space, 'Space not found');
+    return space;
+  }
+
   /**
    * Clears storage - doesn't work for now.
    */
@@ -141,7 +147,7 @@ export class MetadataStore {
   async addSpace(record: SpaceMetadata) {
     assert(
       !(this._metadata.spaces ?? []).find((space) => space.key === record.key),
-      'Cannot overwrite existing space in metadata'
+      'Cannot overwrite existing space in metadata',
     );
 
     (this._metadata.spaces ??= []).push(record);
@@ -149,25 +155,17 @@ export class MetadataStore {
   }
 
   async setSpaceLatestTimeframe(spaceKey: PublicKey, timeframe: Timeframe) {
-    const space = this.spaces.find((space) => space.key === spaceKey);
-    assert(space, 'Space not found');
-
-    space.dataTimeframe = timeframe;
+    this._getSpace(spaceKey).dataTimeframe = timeframe;
     await this._save();
   }
 
-  async setSpaceSnapshot(spaceKey: PublicKey, snapshot: string) {
-    const space = this.spaces.find((space) => space.key === spaceKey);
-    assert(space, 'Space not found');
-
-    space.snapshot = snapshot;
+  async setCache(spaceKey: PublicKey, cache: SpaceCache) {
+    this._getSpace(spaceKey).cache = cache;
     await this._save();
   }
 
   async setWritableFeedKeys(spaceKey: PublicKey, controlFeedKey: PublicKey, dataFeedKey: PublicKey) {
-    const space = this.spaces.find((space) => space.key === spaceKey);
-    assert(space, 'Space not found');
-
+    const space = this._getSpace(spaceKey);
     space.controlFeedKey = controlFeedKey;
     space.dataFeedKey = dataFeedKey;
     await this._save();

package/src/pipeline/message-selector.ts

@@ -23,7 +23,7 @@ export const createMessageSelector = (timeframeClock: TimeframeClock): FeedBlock
     // Pick the first candidate with a valid timeframe that has no gaps.
     for (let i = 0; i < messages.length; i++) {
       const {
-        data: { timeframe }
+        data: { timeframe },
       } = messages[i];
       assert(timeframe);
 

package/src/pipeline/pipeline.test.ts

@@ -7,7 +7,6 @@ import * as fc from 'fast-check';
 import { inspect } from 'util';
 
 import { asyncTimeout } from '@dxos/async';
-import { checkType } from '@dxos/debug';
 import { FeedStore, FeedWrapper } from '@dxos/feed-store';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
@@ -20,10 +19,14 @@ import { range } from '@dxos/util';
 import { TestFeedBuilder } from '../testing';
 import { Pipeline } from './pipeline';
 
+const TEST_MESSAGE: FeedMessage = {
+  timeframe: new Timeframe(),
+  payload: {},
+};
+
 describe('pipeline/Pipeline', () => {
   test('asynchronous reader & writer without ordering', async () => {
-    const pipeline = new Pipeline(new Timeframe());
-    afterTest(() => pipeline.stop());
+    const pipeline = new Pipeline();
 
     const builder = new TestFeedBuilder();
     const feedStore = builder.createFeedStore();
@@ -31,29 +34,14 @@ describe('pipeline/Pipeline', () => {
     // Remote feeds from other peers.
     const numFeeds = 5;
     const messagesPerFeed = 10;
-    for (const feedIdx in range(numFeeds)) {
+    for (const _ in range(numFeeds)) {
       const key = await builder.keyring.createKey();
       const feed = await feedStore.openFeed(key, { writable: true });
       void pipeline.addFeed(feed);
 
       setTimeout(async () => {
-        for (const msgIdx in range(messagesPerFeed)) {
-          await feed.append(
-            checkType<FeedMessage>({
-              timeframe: new Timeframe(),
-              payload: {
-                data: {
-                  batch: {
-                    objects: [
-                      {
-                        objectId: `${feedIdx}-${msgIdx}`
-                      }
-                    ]
-                  }
-                }
-              }
-            })
-          );
+        for (const _ in range(messagesPerFeed)) {
+          await feed.append(TEST_MESSAGE);
         }
       });
     }
@@ -64,20 +52,12 @@ describe('pipeline/Pipeline', () => {
     void pipeline.addFeed(feed);
     pipeline.setWriteFeed(feed);
 
-    for (const msgIdx in range(messagesPerFeed)) {
-      await pipeline.writer!.write({
-        data: {
-          batch: {
-            objects: [
-              {
-                objectId: `local-${msgIdx}`
-              }
-            ]
-          }
-        }
-      });
+    for (const _ in range(messagesPerFeed)) {
+      await pipeline.writer!.write({});
     }
 
+    await pipeline.start();
+
     let msgCount = 0;
     for await (const _ of pipeline.consume()) {
       if (++msgCount === numFeeds * messagesPerFeed) {
@@ -86,6 +66,49 @@ describe('pipeline/Pipeline', () => {
     }
   });
 
+  test('reading and writing with cursor changes', async () => {
+    const pipeline = new Pipeline();
+    afterTest(() => pipeline.stop());
+
+    const builder = new TestFeedBuilder();
+    const feedStore = builder.createFeedStore();
+    const key = await builder.keyring.createKey();
+    const feed = await feedStore.openFeed(key, { writable: true });
+    await pipeline.addFeed(feed);
+
+    const numMessages = 30;
+    const sequenceNumbers: number[] = [];
+    for (const _ of range(numMessages)) {
+      const { seq } = await feed.appendWithReceipt(TEST_MESSAGE);
+      sequenceNumbers.push(seq);
+    }
+
+    const processedSequenceNumbers: number[] = [];
+
+    // skip first 10, process the rest, and the repeat the last 10.
+    const expectedSequenceNumbers = [...sequenceNumbers.slice(10, 30), ...sequenceNumbers.slice(20, 30)];
+
+    await pipeline.setCursor(new Timeframe([[feed.key, 9]]));
+    await pipeline.start();
+
+    for await (const block of pipeline.consume()) {
+      processedSequenceNumbers.push(block.seq);
+
+      if (processedSequenceNumbers.length === 20) {
+        // not awaited to avoid a deadlock.
+        void pipeline.pause().then(async () => {
+          await pipeline.setCursor(new Timeframe([[feed.key, 19]]));
+          await pipeline.unpause();
+        });
+      }
+      if (processedSequenceNumbers.length === 30) {
+        void pipeline.stop();
+      }
+    }
+
+    expect(processedSequenceNumbers).toEqual(expectedSequenceNumbers);
+  });
+
   test
     .skip('stress', async () => {
       const builder = new TestFeedBuilder();
@@ -108,7 +131,8 @@ describe('pipeline/Pipeline', () => {
         }
 
         async start() {
-          this.pipeline = new Pipeline(this.startingTimeframe);
+          this.pipeline = new Pipeline();
+          await this.pipeline.setCursor(this.startingTimeframe);
 
           await this.pipeline.start();
 
@@ -188,7 +212,7 @@ describe('pipeline/Pipeline', () => {
                 console.log('empty endtimeframe', {
                   id: agent.id,
                   endTimeframe: agent.pipeline.state.endTimeframe,
-                  feeds: agent.pipeline.getFeeds().map((feed) => [feed.key.toString(), feed.length])
+                  feeds: agent.pipeline.getFeeds().map((feed) => [feed.key.toString(), feed.length]),
                 });
               }
               await asyncTimeout(agent.pipeline.state.waitUntilTimeframe(agent.pipeline.state.endTimeframe), 1000);
@@ -212,15 +236,15 @@ describe('pipeline/Pipeline', () => {
                     messages: agent.messages.length,
                     feeds: agent.pipeline.getFeeds().map((feed) => [feed.key, feed.length]),
                     timeframe: agent.pipeline.state.timeframe,
-                    endTimeframe: agent.pipeline.state.endTimeframe
+                    endTimeframe: agent.pipeline.state.endTimeframe,
                   })),
                   feeds: real.feedStore.feeds.map((feed) => [feed.key, feed.length]),
-                  targets
+                  targets,
                 },
                 false,
                 null,
-                true
-              )
+                true,
+              ),
             );
             throw err;
           }
@@ -254,9 +278,9 @@ describe('pipeline/Pipeline', () => {
         [
           fc.tuple(anAgentId, fc.integer({ min: 1, max: 10 })).map(([agent, count]) => new WriteCommand(agent, count)),
           fc.constant(new SyncCommand()),
-          anAgentId.map((agent) => new RestartCommand(agent))
+          anAgentId.map((agent) => new RestartCommand(agent)),
         ],
-        { size: 'large' }
+        { size: 'large' },
       );
 
       const model = fc.asyncProperty(commands, async (commands) => {
@@ -270,8 +294,8 @@ describe('pipeline/Pipeline', () => {
           model: {},
           real: {
             feedStore,
-            agents
-          }
+            agents,
+          },
         });
 
         try {
@@ -284,7 +308,7 @@ describe('pipeline/Pipeline', () => {
 
       const examples: [commands: Iterable<fc.AsyncCommand<Model, Real, boolean>>][] = [
         [[new WriteCommand(agentIds[0], 10), new WriteCommand(agentIds[1], 10), new SyncCommand()]],
-        [[new WriteCommand(agentIds[0], 4), new RestartCommand(agentIds[0]), new SyncCommand()]]
+        [[new WriteCommand(agentIds[0], 4), new RestartCommand(agentIds[0]), new SyncCommand()]],
       ];
 
       await fc.assert(model, { examples });