@dxos/echo-pipeline 0.1.31-next.ea8f0bc → 0.1.32-next.77d513e

package/dist/lib/node/testing/index.cjs

@@ -48,7 +48,7 @@ var import_messaging = require("@dxos/messaging");
 var import_model_factory = require("@dxos/model-factory");
 var import_network_manager2 = require("@dxos/network-manager");
 var import_random_access_storage = require("@dxos/random-access-storage");
-var import_teleport_extension_presence = require("@dxos/teleport-extension-presence");
+var import_teleport_extension_gossip = require("@dxos/teleport-extension-gossip");
 var import_util7 = require("@dxos/util");
 
 // packages/core/echo/echo-pipeline/src/dbhost/data-service-host.ts
@@ -75,56 +75,53 @@ var DataServiceHost = class {
           objects
         }
       });
-      this._itemDemuxer.mutation.on(ctx, (mutation) => {
+      this._itemDemuxer.mutation.on(ctx, (message) => {
         var _a;
+        const { batch, meta } = message;
         (0, import_log.log)("Object update", {
-          mutation
+          batch,
+          meta
         }, {
           file: "data-service-host.ts",
-          line: 49,
+          line: 50,
           scope: this,
           callSite: (f, a) => f(...a)
         });
         const clientTag = this._clientTagMap.get([
-          mutation.meta.feedKey,
-          mutation.meta.seq
+          message.meta.feedKey,
+          message.meta.seq
         ]);
-        const batch = {
-          objects: [
-            {
-              ...mutation.data,
-              mutations: (_a = mutation.data.mutations) == null ? void 0 : _a.map((m, mutationIdx) => ({
-                ...m,
-                meta: mutation.meta
-              })),
-              meta: mutation.meta
-            }
-          ]
-        };
+        (_a = batch.objects) == null ? void 0 : _a.forEach((object) => {
+          (0, import_echo_db.setMetadataOnObject)(object, meta);
+        });
         if (clientTag) {
           (0, import_echo_db.tagMutationsInBatch)(batch, clientTag);
         }
         next({
           clientTag,
-          feedKey: mutation.meta.feedKey,
-          seq: mutation.meta.seq,
+          feedKey: message.meta.feedKey,
+          seq: message.meta.seq,
           batch
         });
       });
     });
   }
   async write(request) {
-    var _a, _b;
+    var _a;
     (0, import_node_assert.default)(this._writeStream, "Cannot write mutations in readonly mode");
-    (0, import_node_assert.default)(((_a = request.batch.objects) == null ? void 0 : _a.length) === 1, "Only single object mutations are supported");
     const receipt = await this._writeStream.write({
-      object: {
-        ...request.batch.objects[0],
-        mutations: (_b = request.batch.objects[0].mutations) == null ? void 0 : _b.map((m) => ({
-          ...m,
-          meta: void 0
-        })),
-        meta: void 0
+      batch: {
+        objects: (_a = request.batch.objects) == null ? void 0 : _a.map((object) => {
+          var _a2;
+          return {
+            ...object,
+            mutations: (_a2 = object.mutations) == null ? void 0 : _a2.map((m) => ({
+              ...m,
+              meta: void 0
+            })),
+            meta: void 0
+          };
+        })
       }
     });
     if (request.clientTag) {
@@ -382,7 +379,7 @@ var MetadataStore = class {
         size: dataSize
       }, {
         file: "metadata-store.ts",
-        line: 68,
+        line: 67,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -398,7 +395,7 @@ var MetadataStore = class {
         err
       }, {
         file: "metadata-store.ts",
-        line: 80,
+        line: 79,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -423,7 +420,7 @@ var MetadataStore = class {
         size: encoded.length
       }, {
         file: "metadata-store.ts",
-        line: 103,
+        line: 102,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -438,7 +435,7 @@ var MetadataStore = class {
   async clear() {
     (0, import_log3.log)("clearing all metadata", {}, {
       file: "metadata-store.ts",
-      line: 116,
+      line: 115,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -588,6 +585,7 @@ var import_util4 = require("@dxos/util");
 // packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts
 var import_node_assert7 = __toESM(require("node:assert"));
 var import_async5 = require("@dxos/async");
+var import_context2 = require("@dxos/context");
 var import_feed_store = require("@dxos/feed-store");
 var import_keys2 = require("@dxos/keys");
 var import_log6 = require("@dxos/log");
@@ -722,7 +720,7 @@ var PipelineState = class {
     return this._timeframeClock.timeframe;
   }
   get targetTimeframe() {
-    return this._targetTimeframe ? import_timeframe2.Timeframe.merge(this.endTimeframe, this._targetTimeframe) : this.endTimeframe;
+    return this._targetTimeframe ? this._targetTimeframe : new import_timeframe2.Timeframe();
   }
   async waitUntilTimeframe(target) {
     await this._timeframeClock.waitUntilReached(target);
@@ -737,7 +735,7 @@ var PipelineState = class {
   *
   * @param timeout Timeout in milliseconds to specify the maximum wait time.
   */
-  async waitUntilReachedTargetTimeframe({ timeout } = {}) {
+  async waitUntilReachedTargetTimeframe({ ctx = new import_context2.Context(), timeout, breakOnStall = true } = {}) {
     var _a;
     (0, import_log6.log)("waitUntilReachedTargetTimeframe", {
       timeout,
@@ -745,7 +743,7 @@ var PipelineState = class {
       target: this.targetTimeframe
     }, {
       file: "pipeline.ts",
-      line: 82,
+      line: 96,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -753,11 +751,14 @@ var PipelineState = class {
       this._timeframeClock.update.waitForCondition(() => {
         return import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe).isEmpty();
       }),
-      this._iterator.stalled.discardParameter().waitForCount(1)
+      ...breakOnStall ? [
+        this._iterator.stalled.discardParameter().waitForCount(1)
+      ] : []
     ]);
     let done = false;
     if (timeout) {
       return Promise.race([
+        (0, import_context2.rejectOnDispose)(ctx),
         this._reachedTargetPromise.then(() => {
           done = true;
         }),
@@ -772,7 +773,7 @@ var PipelineState = class {
             dependencies: import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe)
           }, {
             file: "pipeline.ts",
-            line: 107,
+            line: 122,
             scope: this,
             callSite: (f, a) => f(...a)
           });
@@ -798,7 +799,7 @@ var Pipeline = class {
     this._feedSetIterator.stalled.on((iterator) => {
       import_log6.log.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, {}, {
         file: "pipeline.ts",
-        line: 183,
+        line: 198,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -828,14 +829,14 @@ var Pipeline = class {
   async start() {
     (0, import_log6.log)("starting...", {}, {
       file: "pipeline.ts",
-      line: 218,
+      line: 233,
       scope: this,
       callSite: (f, a) => f(...a)
     });
     await this._feedSetIterator.open();
     (0, import_log6.log)("started", {}, {
       file: "pipeline.ts",
-      line: 220,
+      line: 235,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -843,7 +844,7 @@ var Pipeline = class {
   async stop() {
     (0, import_log6.log)("stopping...", {}, {
       file: "pipeline.ts",
-      line: 224,
+      line: 239,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -851,7 +852,7 @@ var Pipeline = class {
     await this._processingTrigger.wait();
     (0, import_log6.log)("stopped", {}, {
       file: "pipeline.ts",
-      line: 227,
+      line: 242,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -1029,7 +1030,7 @@ var Space = class Space2 {
         credential
       }, {
         file: "space.ts",
-        line: 74,
+        line: 79,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -1092,7 +1093,7 @@ var Space = class Space2 {
   async open() {
     (0, import_log8.log)("opening...", {}, {
       file: "space.ts",
-      line: 145,
+      line: 149,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -1104,18 +1105,17 @@ var Space = class Space2 {
     this._isOpen = true;
     (0, import_log8.log)("opened", {}, {
       file: "space.ts",
-      line: 155,
+      line: 159,
       scope: this,
       callSite: (f, a) => f(...a)
     });
   }
   async close() {
-    var _a;
     (0, import_log8.log)("closing...", {
       key: this._key
     }, {
       file: "space.ts",
-      line: 160,
+      line: 164,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -1123,37 +1123,30 @@ var Space = class Space2 {
       return;
     }
     await this.protocol.stop();
-    await ((_a = this._dataPipelineController) == null ? void 0 : _a.close());
     await this._controlPipeline.stop();
     this._isOpen = false;
     (0, import_log8.log)("closed", {}, {
       file: "space.ts",
-      line: 171,
+      line: 174,
       scope: this,
       callSite: (f, a) => f(...a)
     });
   }
-  async initDataPipeline(controller) {
+  // TODO(dmaretskyi): Make reusable.
+  async createDataPipeline({ start }) {
     (0, import_node_assert8.default)(this._isOpen, "Space must be open to initialize data pipeline.");
-    (0, import_node_assert8.default)(!this._dataPipelineController, "Data pipeline already initialized.");
     (0, import_node_assert8.default)(!this._dataPipeline, "Data pipeline already initialized.");
-    this._dataPipelineController = controller;
-    await this._dataPipelineController.open({
-      openPipeline: async (start) => {
-        (0, import_node_assert8.default)(!this._dataPipeline, "Data pipeline already initialized.");
-        this._dataPipeline = new Pipeline(start);
-        if (this._dataFeed) {
-          this._dataPipeline.setWriteFeed(this._dataFeed);
-        }
-        for (const feed of this._controlPipeline.spaceState.feeds.values()) {
-          if (feed.assertion.designation === import_credentials3.AdmittedFeed.Designation.DATA) {
-            await this._dataPipeline.addFeed(await this._feedProvider(feed.key));
-          }
-        }
-        await this._dataPipeline.start();
-        return this._dataPipeline;
+    const pipeline = new Pipeline(start);
+    if (this._dataFeed) {
+      pipeline.setWriteFeed(this._dataFeed);
+    }
+    this._dataPipeline = pipeline;
+    for (const feed of this._controlPipeline.spaceState.feeds.values()) {
+      if (feed.assertion.designation === import_credentials3.AdmittedFeed.Designation.DATA) {
+        await pipeline.addFeed(await this._feedProvider(feed.key));
       }
-    });
+    }
+    return pipeline;
   }
 };
 __decorate4([
@@ -1432,10 +1425,10 @@ var import_credentials4 = require("@dxos/credentials");
 var import_debug5 = require("@dxos/debug");
 var import_credentials5 = require("@dxos/protocols/proto/dxos/halo/credentials");
 
-// packages/core/echo/echo-pipeline/src/space/data-pipeline-controller.ts
+// packages/core/echo/echo-pipeline/src/space/data-pipeline.ts
 var import_node_assert9 = __toESM(require("node:assert"));
 var import_async8 = require("@dxos/async");
-var import_context2 = require("@dxos/context");
+var import_context3 = require("@dxos/context");
 var import_debug6 = require("@dxos/debug");
 var import_echo_db3 = require("@dxos/echo-db");
 var import_log11 = require("@dxos/log");
@@ -1453,13 +1446,18 @@ var __decorate7 = function(decorators, target, key, desc) {
 var MESSAGES_PER_SNAPSHOT = 10;
 var AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL = 5e3;
 var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
-var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
+var DISABLE_SNAPSHOT_CACHE = false;
+var DataPipeline = class DataPipeline2 {
   constructor(_params) {
     this._params = _params;
-    this._ctx = new import_context2.Context();
+    this._ctx = new import_context3.Context();
     this._lastAutomaticSnapshotTimeframe = new import_timeframe3.Timeframe();
+    this._isOpen = false;
     this.onTimeframeReached = new import_async8.Event();
   }
+  get isOpen() {
+    return this._isOpen;
+  }
   get pipelineState() {
     var _a;
     return (_a = this._pipeline) == null ? void 0 : _a.state;
@@ -1472,14 +1470,25 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
     var _a;
     return snapshotTimeframeToStartingTimeframe((_a = this.snapshotTimeframe) != null ? _a : new import_timeframe3.Timeframe());
   }
+  setTargetTimeframe(timeframe) {
+    var _a;
+    this._targetTimeframe = timeframe;
+    (_a = this._pipeline) == null ? void 0 : _a.state.setTargetTimeframe(timeframe);
+  }
   async open(spaceContext) {
     var _a, _b, _c, _d;
+    if (this._isOpen) {
+      return;
+    }
     this._spaceContext = spaceContext;
     if (this._params.snapshotId) {
       this._snapshot = await this._params.snapshotManager.load(this._params.snapshotId);
       this._lastAutomaticSnapshotTimeframe = (_b = (_a = this._snapshot) == null ? void 0 : _a.timeframe) != null ? _b : new import_timeframe3.Timeframe();
     }
     this._pipeline = await this._spaceContext.openPipeline(this.getStartTimeframe());
+    if (this._targetTimeframe) {
+      this._pipeline.state.setTargetTimeframe(this._targetTimeframe);
+    }
     const feedWriter = createMappedFeedWriter((data) => ({
       data
     }), (_c = this._pipeline.writer) != null ? _c : (0, import_debug6.failUndefined)());
@@ -1490,30 +1499,33 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
       await this._consumePipeline();
     });
     this._createPeriodicSnapshots();
+    this._isOpen = true;
   }
   _createPeriodicSnapshots() {
     this.onTimeframeReached.debounce(TIMEFRAME_SAVE_DEBOUNCE_INTERVAL).on(this._ctx, async () => {
       await this._saveLatestTimeframe();
     });
-    this.onTimeframeReached.debounce(AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL).on(this._ctx, async () => {
-      var _a, _b;
-      const latestTimeframe = (_a = this._pipeline) == null ? void 0 : _a.state.timeframe;
-      if (!latestTimeframe) {
-        return;
-      }
-      if (latestTimeframe.totalMessages() - this._lastAutomaticSnapshotTimeframe.totalMessages() > MESSAGES_PER_SNAPSHOT) {
-        const snapshot = await this._saveSnapshot();
-        this._lastAutomaticSnapshotTimeframe = (_b = snapshot.timeframe) != null ? _b : (0, import_debug6.failUndefined)();
-        (0, import_log11.log)("save", {
-          snapshot
-        }, {
-          file: "data-pipeline-controller.ts",
-          line: 149,
-          scope: this,
-          callSite: (f, a) => f(...a)
-        });
-      }
-    });
+    if (!DISABLE_SNAPSHOT_CACHE) {
+      this.onTimeframeReached.debounce(AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL).on(this._ctx, async () => {
+        var _a, _b;
+        const latestTimeframe = (_a = this._pipeline) == null ? void 0 : _a.state.timeframe;
+        if (!latestTimeframe) {
+          return;
+        }
+        if (latestTimeframe.totalMessages() - this._lastAutomaticSnapshotTimeframe.totalMessages() > MESSAGES_PER_SNAPSHOT) {
+          const snapshot = await this._saveSnapshot();
+          this._lastAutomaticSnapshotTimeframe = (_b = snapshot.timeframe) != null ? _b : (0, import_debug6.failUndefined)();
+          (0, import_log11.log)("save", {
+            snapshot
+          }, {
+            file: "data-pipeline.ts",
+            line: 161,
+            scope: this,
+            callSite: (f, a) => f(...a)
+          });
+        }
+      });
+    }
   }
   async _saveSnapshot() {
     const snapshot = await this.createSnapshot();
@@ -1522,21 +1534,41 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
     return snapshot;
   }
   async _saveLatestTimeframe() {
-    var _a;
+    var _a, _b;
     const latestTimeframe = (_a = this._pipeline) == null ? void 0 : _a.state.timeframe;
+    (0, import_log11.log)("save latest timeframe", {
+      latestTimeframe,
+      spaceKey: this._params.spaceKey
+    }, {
+      file: "data-pipeline.ts",
+      line: 176,
+      scope: this,
+      callSite: (f, a) => f(...a)
+    });
     if (latestTimeframe) {
-      await this._params.metadataStore.setSpaceLatestTimeframe(this._params.spaceKey, latestTimeframe);
+      const newTimeframe = import_timeframe3.Timeframe.merge((_b = this._targetTimeframe) != null ? _b : new import_timeframe3.Timeframe(), latestTimeframe);
+      await this._params.metadataStore.setSpaceLatestTimeframe(this._params.spaceKey, newTimeframe);
     }
   }
   async close() {
     var _a, _b, _c;
+    if (!this._isOpen) {
+      return;
+    }
+    (0, import_log11.log)("close", {}, {
+      file: "data-pipeline.ts",
+      line: 188,
+      scope: this,
+      callSite: (f, a) => f(...a)
+    });
+    this._isOpen = false;
     try {
       await this._saveLatestTimeframe();
       await this._saveSnapshot();
     } catch (err) {
       import_log11.log.catch(err, {}, {
-        file: "data-pipeline-controller.ts",
-        line: 173,
+        file: "data-pipeline.ts",
+        line: 195,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -1549,6 +1581,7 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
   }
   createSnapshot() {
     var _a, _b;
+    (0, import_node_assert9.default)(this.databaseBackend, "Database backend is not initialized.");
     return {
       spaceKey: this._params.spaceKey.asUint8Array(),
       timeframe: (_b = (_a = this._pipeline) == null ? void 0 : _a.state.timeframe) != null ? _b : new import_timeframe3.Timeframe(),
@@ -1562,8 +1595,8 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
       (0, import_log11.log)("processing message", {
         msg
       }, {
-        file: "data-pipeline-controller.ts",
-        line: 194,
+        file: "data-pipeline.ts",
+        line: 217,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -1574,15 +1607,15 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
             import_log11.log.error("Could not find feed.", {
               feedKey
             }, {
-              file: "data-pipeline-controller.ts",
-              line: 200,
+              file: "data-pipeline.ts",
+              line: 223,
               scope: this,
               callSite: (f, a) => f(...a)
             });
             continue;
           }
           await this.databaseBackend.echoProcessor({
-            data: data.payload.data.object,
+            batch: data.payload.data.batch,
             meta: {
               feedKey,
               seq,
@@ -1594,8 +1627,8 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
         }
       } catch (err) {
         import_log11.log.catch(err, {}, {
-          file: "data-pipeline-controller.ts",
-          line: 216,
+          file: "data-pipeline.ts",
+          line: 239,
           scope: this,
           callSite: (f, a) => f(...a)
         });
@@ -1607,9 +1640,15 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
     await this._pipeline.state.waitUntilTimeframe(timeframe);
   }
 };
-DataPipelineControllerImpl = __decorate7([
+__decorate7([
+  import_async8.synchronized
+], DataPipeline.prototype, "open", null);
+__decorate7([
+  import_async8.synchronized
+], DataPipeline.prototype, "close", null);
+DataPipeline = __decorate7([
   (0, import_async8.trackLeaks)("open", "close")
-], DataPipelineControllerImpl);
+], DataPipeline);
 var snapshotTimeframeToStartingTimeframe = (snapshotTimeframe) => {
   return snapshotTimeframe.map(([key, seq]) => [
     key,
@@ -1709,7 +1748,7 @@ var TestAgent = class {
     await metadataStore.addSpace({
       key: spaceKey
     });
-    const dataPipelineController = new DataPipelineControllerImpl({
+    const dataPipelineController = new DataPipeline({
       modelFactory: new import_model_factory.ModelFactory().registerModel(import_document_model.DocumentModel),
       metadataStore,
       snapshotManager,
@@ -1725,14 +1764,22 @@ var TestAgent = class {
       feedProvider: (feedKey) => this.feedStore.openFeed(feedKey)
     }).setControlFeed(controlFeed).setDataFeed(dataFeed);
     await space.open();
-    await space.initDataPipeline(dataPipelineController);
+    await dataPipelineController.open({
+      openPipeline: async (start) => {
+        const pipeline = await space.createDataPipeline({
+          start
+        });
+        await pipeline.start();
+        return pipeline;
+      }
+    });
     this._spaces.set(spaceKey, space);
     return [
       space,
       dataPipelineController
     ];
   }
-  createSpaceProtocol(topic, presence) {
+  createSpaceProtocol(topic, gossip) {
     return new SpaceProtocol({
       topic,
       swarmIdentity: {
@@ -1742,18 +1789,23 @@ var TestAgent = class {
       },
       networkManager: this._networkManagerProvider(),
       onSessionAuth: (session) => {
-        session.addExtension("dxos.mesh.teleport.presence", (presence != null ? presence : this.createPresence()).createExtension({
+        session.addExtension("dxos.mesh.teleport.gossip", (gossip != null ? gossip : this.createGossip()).createExtension({
           remotePeerId: session.remotePeerId
         }));
       }
     });
   }
-  createPresence() {
-    return new import_teleport_extension_presence.Presence({
-      localPeerId: this.deviceKey,
+  createGossip() {
+    return new import_teleport_extension_gossip.Gossip({
+      localPeerId: this.deviceKey
+    });
+  }
+  createPresence(gossip) {
+    return new import_teleport_extension_gossip.Presence({
       announceInterval: 30,
       offlineTimeout: 200,
-      identityKey: this.identityKey
+      identityKey: this.identityKey,
+      gossip: gossip != null ? gossip : this.createGossip()
     });
   }
 };
@@ -1770,7 +1822,7 @@ var createMemoryDatabase = async (modelFactory) => {
   const feed = new import_testing2.MockFeedWriter();
   const backend = new DatabaseBackendHost(feed, void 0);
   feed.written.on(([data, meta]) => backend.echoProcessor({
-    data: data.object,
+    batch: data.batch,
     meta: {
       ...meta,
       memberKey: import_keys7.PublicKey.random(),
@@ -1806,14 +1858,18 @@ var testLocalDatabase = async (create, check = create) => {
   var _a;
   const objectId = import_keys7.PublicKey.random().toHex();
   await ((_a = create.databaseBackend.getWriteStream()) == null ? void 0 : _a.write({
-    object: {
-      objectId,
-      genesis: {
-        modelType: import_document_model2.DocumentModel.meta.type
-      }
+    batch: {
+      objects: [
+        {
+          objectId,
+          genesis: {
+            modelType: import_document_model2.DocumentModel.meta.type
+          }
+        }
+      ]
     }
   }));
-  await (0, import_async9.asyncTimeout)(check._itemManager.update.waitForCondition(() => check._itemManager.entities.has(objectId)), 500);
+  await (0, import_async9.asyncTimeout)(check.databaseBackend._itemDemuxer.mutation.waitForCondition(() => check._itemManager.entities.has(objectId)), 500);
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {