@dxos/echo-pipeline 0.1.31 → 0.1.32-next.77d513e

package/dist/lib/node/testing/index.cjs

@@ -379,7 +379,7 @@ var MetadataStore = class {
         size: dataSize
       }, {
         file: "metadata-store.ts",
-        line: 68,
+        line: 67,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -395,7 +395,7 @@ var MetadataStore = class {
         err
       }, {
         file: "metadata-store.ts",
-        line: 80,
+        line: 79,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -420,7 +420,7 @@ var MetadataStore = class {
         size: encoded.length
       }, {
         file: "metadata-store.ts",
-        line: 103,
+        line: 102,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -435,7 +435,7 @@ var MetadataStore = class {
   async clear() {
     (0, import_log3.log)("clearing all metadata", {}, {
       file: "metadata-store.ts",
-      line: 116,
+      line: 115,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -585,6 +585,7 @@ var import_util4 = require("@dxos/util");
 // packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts
 var import_node_assert7 = __toESM(require("node:assert"));
 var import_async5 = require("@dxos/async");
+var import_context2 = require("@dxos/context");
 var import_feed_store = require("@dxos/feed-store");
 var import_keys2 = require("@dxos/keys");
 var import_log6 = require("@dxos/log");
@@ -719,7 +720,7 @@ var PipelineState = class {
     return this._timeframeClock.timeframe;
   }
   get targetTimeframe() {
-    return this._targetTimeframe ? import_timeframe2.Timeframe.merge(this.endTimeframe, this._targetTimeframe) : this.endTimeframe;
+    return this._targetTimeframe ? this._targetTimeframe : new import_timeframe2.Timeframe();
   }
   async waitUntilTimeframe(target) {
     await this._timeframeClock.waitUntilReached(target);
@@ -734,7 +735,7 @@ var PipelineState = class {
   *
   * @param timeout Timeout in milliseconds to specify the maximum wait time.
   */
-  async waitUntilReachedTargetTimeframe({ timeout } = {}) {
+  async waitUntilReachedTargetTimeframe({ ctx = new import_context2.Context(), timeout, breakOnStall = true } = {}) {
     var _a;
     (0, import_log6.log)("waitUntilReachedTargetTimeframe", {
       timeout,
@@ -742,7 +743,7 @@ var PipelineState = class {
       target: this.targetTimeframe
     }, {
       file: "pipeline.ts",
-      line: 82,
+      line: 96,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -750,11 +751,14 @@ var PipelineState = class {
       this._timeframeClock.update.waitForCondition(() => {
         return import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe).isEmpty();
       }),
-      this._iterator.stalled.discardParameter().waitForCount(1)
+      ...breakOnStall ? [
+        this._iterator.stalled.discardParameter().waitForCount(1)
+      ] : []
     ]);
     let done = false;
     if (timeout) {
       return Promise.race([
+        (0, import_context2.rejectOnDispose)(ctx),
         this._reachedTargetPromise.then(() => {
           done = true;
         }),
@@ -769,7 +773,7 @@ var PipelineState = class {
             dependencies: import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe)
           }, {
             file: "pipeline.ts",
-            line: 107,
+            line: 122,
             scope: this,
             callSite: (f, a) => f(...a)
           });
@@ -795,7 +799,7 @@ var Pipeline = class {
     this._feedSetIterator.stalled.on((iterator) => {
       import_log6.log.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, {}, {
         file: "pipeline.ts",
-        line: 183,
+        line: 198,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -825,14 +829,14 @@ var Pipeline = class {
   async start() {
     (0, import_log6.log)("starting...", {}, {
       file: "pipeline.ts",
-      line: 218,
+      line: 233,
       scope: this,
       callSite: (f, a) => f(...a)
     });
     await this._feedSetIterator.open();
     (0, import_log6.log)("started", {}, {
       file: "pipeline.ts",
-      line: 220,
+      line: 235,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -840,7 +844,7 @@ var Pipeline = class {
   async stop() {
     (0, import_log6.log)("stopping...", {}, {
       file: "pipeline.ts",
-      line: 224,
+      line: 239,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -848,7 +852,7 @@ var Pipeline = class {
     await this._processingTrigger.wait();
     (0, import_log6.log)("stopped", {}, {
       file: "pipeline.ts",
-      line: 227,
+      line: 242,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -1026,7 +1030,7 @@ var Space = class Space2 {
         credential
       }, {
         file: "space.ts",
-        line: 74,
+        line: 79,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -1089,7 +1093,7 @@ var Space = class Space2 {
   async open() {
     (0, import_log8.log)("opening...", {}, {
       file: "space.ts",
-      line: 145,
+      line: 149,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -1101,18 +1105,17 @@ var Space = class Space2 {
     this._isOpen = true;
     (0, import_log8.log)("opened", {}, {
       file: "space.ts",
-      line: 155,
+      line: 159,
       scope: this,
       callSite: (f, a) => f(...a)
     });
   }
   async close() {
-    var _a;
     (0, import_log8.log)("closing...", {
       key: this._key
     }, {
       file: "space.ts",
-      line: 160,
+      line: 164,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -1120,37 +1123,30 @@ var Space = class Space2 {
       return;
     }
     await this.protocol.stop();
-    await ((_a = this._dataPipelineController) == null ? void 0 : _a.close());
     await this._controlPipeline.stop();
     this._isOpen = false;
     (0, import_log8.log)("closed", {}, {
       file: "space.ts",
-      line: 171,
+      line: 174,
       scope: this,
       callSite: (f, a) => f(...a)
     });
   }
-  async initDataPipeline(controller) {
+  // TODO(dmaretskyi): Make reusable.
+  async createDataPipeline({ start }) {
     (0, import_node_assert8.default)(this._isOpen, "Space must be open to initialize data pipeline.");
-    (0, import_node_assert8.default)(!this._dataPipelineController, "Data pipeline already initialized.");
     (0, import_node_assert8.default)(!this._dataPipeline, "Data pipeline already initialized.");
-    this._dataPipelineController = controller;
-    await this._dataPipelineController.open({
-      openPipeline: async (start) => {
-        (0, import_node_assert8.default)(!this._dataPipeline, "Data pipeline already initialized.");
-        this._dataPipeline = new Pipeline(start);
-        if (this._dataFeed) {
-          this._dataPipeline.setWriteFeed(this._dataFeed);
-        }
-        for (const feed of this._controlPipeline.spaceState.feeds.values()) {
-          if (feed.assertion.designation === import_credentials3.AdmittedFeed.Designation.DATA) {
-            await this._dataPipeline.addFeed(await this._feedProvider(feed.key));
-          }
-        }
-        await this._dataPipeline.start();
-        return this._dataPipeline;
+    const pipeline = new Pipeline(start);
+    if (this._dataFeed) {
+      pipeline.setWriteFeed(this._dataFeed);
+    }
+    this._dataPipeline = pipeline;
+    for (const feed of this._controlPipeline.spaceState.feeds.values()) {
+      if (feed.assertion.designation === import_credentials3.AdmittedFeed.Designation.DATA) {
+        await pipeline.addFeed(await this._feedProvider(feed.key));
       }
-    });
+    }
+    return pipeline;
   }
 };
 __decorate4([
@@ -1429,10 +1425,10 @@ var import_credentials4 = require("@dxos/credentials");
 var import_debug5 = require("@dxos/debug");
 var import_credentials5 = require("@dxos/protocols/proto/dxos/halo/credentials");
 
-// packages/core/echo/echo-pipeline/src/space/data-pipeline-controller.ts
+// packages/core/echo/echo-pipeline/src/space/data-pipeline.ts
 var import_node_assert9 = __toESM(require("node:assert"));
 var import_async8 = require("@dxos/async");
-var import_context2 = require("@dxos/context");
+var import_context3 = require("@dxos/context");
 var import_debug6 = require("@dxos/debug");
 var import_echo_db3 = require("@dxos/echo-db");
 var import_log11 = require("@dxos/log");
@@ -1450,13 +1446,18 @@ var __decorate7 = function(decorators, target, key, desc) {
 var MESSAGES_PER_SNAPSHOT = 10;
 var AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL = 5e3;
 var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
-var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
+var DISABLE_SNAPSHOT_CACHE = false;
+var DataPipeline = class DataPipeline2 {
   constructor(_params) {
     this._params = _params;
-    this._ctx = new import_context2.Context();
+    this._ctx = new import_context3.Context();
     this._lastAutomaticSnapshotTimeframe = new import_timeframe3.Timeframe();
+    this._isOpen = false;
     this.onTimeframeReached = new import_async8.Event();
   }
+  get isOpen() {
+    return this._isOpen;
+  }
   get pipelineState() {
     var _a;
     return (_a = this._pipeline) == null ? void 0 : _a.state;
@@ -1469,14 +1470,25 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
     var _a;
     return snapshotTimeframeToStartingTimeframe((_a = this.snapshotTimeframe) != null ? _a : new import_timeframe3.Timeframe());
   }
+  setTargetTimeframe(timeframe) {
+    var _a;
+    this._targetTimeframe = timeframe;
+    (_a = this._pipeline) == null ? void 0 : _a.state.setTargetTimeframe(timeframe);
+  }
   async open(spaceContext) {
     var _a, _b, _c, _d;
+    if (this._isOpen) {
+      return;
+    }
     this._spaceContext = spaceContext;
     if (this._params.snapshotId) {
       this._snapshot = await this._params.snapshotManager.load(this._params.snapshotId);
       this._lastAutomaticSnapshotTimeframe = (_b = (_a = this._snapshot) == null ? void 0 : _a.timeframe) != null ? _b : new import_timeframe3.Timeframe();
     }
     this._pipeline = await this._spaceContext.openPipeline(this.getStartTimeframe());
+    if (this._targetTimeframe) {
+      this._pipeline.state.setTargetTimeframe(this._targetTimeframe);
+    }
     const feedWriter = createMappedFeedWriter((data) => ({
       data
     }), (_c = this._pipeline.writer) != null ? _c : (0, import_debug6.failUndefined)());
@@ -1487,30 +1499,33 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
       await this._consumePipeline();
     });
     this._createPeriodicSnapshots();
+    this._isOpen = true;
   }
   _createPeriodicSnapshots() {
     this.onTimeframeReached.debounce(TIMEFRAME_SAVE_DEBOUNCE_INTERVAL).on(this._ctx, async () => {
       await this._saveLatestTimeframe();
     });
-    this.onTimeframeReached.debounce(AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL).on(this._ctx, async () => {
-      var _a, _b;
-      const latestTimeframe = (_a = this._pipeline) == null ? void 0 : _a.state.timeframe;
-      if (!latestTimeframe) {
-        return;
-      }
-      if (latestTimeframe.totalMessages() - this._lastAutomaticSnapshotTimeframe.totalMessages() > MESSAGES_PER_SNAPSHOT) {
-        const snapshot = await this._saveSnapshot();
-        this._lastAutomaticSnapshotTimeframe = (_b = snapshot.timeframe) != null ? _b : (0, import_debug6.failUndefined)();
-        (0, import_log11.log)("save", {
-          snapshot
-        }, {
-          file: "data-pipeline-controller.ts",
-          line: 149,
-          scope: this,
-          callSite: (f, a) => f(...a)
-        });
-      }
-    });
+    if (!DISABLE_SNAPSHOT_CACHE) {
+      this.onTimeframeReached.debounce(AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL).on(this._ctx, async () => {
+        var _a, _b;
+        const latestTimeframe = (_a = this._pipeline) == null ? void 0 : _a.state.timeframe;
+        if (!latestTimeframe) {
+          return;
+        }
+        if (latestTimeframe.totalMessages() - this._lastAutomaticSnapshotTimeframe.totalMessages() > MESSAGES_PER_SNAPSHOT) {
+          const snapshot = await this._saveSnapshot();
+          this._lastAutomaticSnapshotTimeframe = (_b = snapshot.timeframe) != null ? _b : (0, import_debug6.failUndefined)();
+          (0, import_log11.log)("save", {
+            snapshot
+          }, {
+            file: "data-pipeline.ts",
+            line: 161,
+            scope: this,
+            callSite: (f, a) => f(...a)
+          });
+        }
+      });
+    }
   }
   async _saveSnapshot() {
     const snapshot = await this.createSnapshot();
@@ -1519,21 +1534,41 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
     return snapshot;
   }
   async _saveLatestTimeframe() {
-    var _a;
+    var _a, _b;
     const latestTimeframe = (_a = this._pipeline) == null ? void 0 : _a.state.timeframe;
+    (0, import_log11.log)("save latest timeframe", {
+      latestTimeframe,
+      spaceKey: this._params.spaceKey
+    }, {
+      file: "data-pipeline.ts",
+      line: 176,
+      scope: this,
+      callSite: (f, a) => f(...a)
+    });
     if (latestTimeframe) {
-      await this._params.metadataStore.setSpaceLatestTimeframe(this._params.spaceKey, latestTimeframe);
+      const newTimeframe = import_timeframe3.Timeframe.merge((_b = this._targetTimeframe) != null ? _b : new import_timeframe3.Timeframe(), latestTimeframe);
+      await this._params.metadataStore.setSpaceLatestTimeframe(this._params.spaceKey, newTimeframe);
     }
   }
   async close() {
     var _a, _b, _c;
+    if (!this._isOpen) {
+      return;
+    }
+    (0, import_log11.log)("close", {}, {
+      file: "data-pipeline.ts",
+      line: 188,
+      scope: this,
+      callSite: (f, a) => f(...a)
+    });
+    this._isOpen = false;
     try {
       await this._saveLatestTimeframe();
       await this._saveSnapshot();
     } catch (err) {
       import_log11.log.catch(err, {}, {
-        file: "data-pipeline-controller.ts",
-        line: 173,
+        file: "data-pipeline.ts",
+        line: 195,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -1546,6 +1581,7 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
   }
   createSnapshot() {
     var _a, _b;
+    (0, import_node_assert9.default)(this.databaseBackend, "Database backend is not initialized.");
     return {
       spaceKey: this._params.spaceKey.asUint8Array(),
       timeframe: (_b = (_a = this._pipeline) == null ? void 0 : _a.state.timeframe) != null ? _b : new import_timeframe3.Timeframe(),
@@ -1559,8 +1595,8 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
       (0, import_log11.log)("processing message", {
         msg
       }, {
-        file: "data-pipeline-controller.ts",
-        line: 194,
+        file: "data-pipeline.ts",
+        line: 217,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -1571,8 +1607,8 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
             import_log11.log.error("Could not find feed.", {
               feedKey
             }, {
-              file: "data-pipeline-controller.ts",
-              line: 200,
+              file: "data-pipeline.ts",
+              line: 223,
               scope: this,
               callSite: (f, a) => f(...a)
             });
@@ -1591,8 +1627,8 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
         }
       } catch (err) {
         import_log11.log.catch(err, {}, {
-          file: "data-pipeline-controller.ts",
-          line: 216,
+          file: "data-pipeline.ts",
+          line: 239,
           scope: this,
           callSite: (f, a) => f(...a)
         });
@@ -1604,9 +1640,15 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
     await this._pipeline.state.waitUntilTimeframe(timeframe);
   }
 };
-DataPipelineControllerImpl = __decorate7([
+__decorate7([
+  import_async8.synchronized
+], DataPipeline.prototype, "open", null);
+__decorate7([
+  import_async8.synchronized
+], DataPipeline.prototype, "close", null);
+DataPipeline = __decorate7([
   (0, import_async8.trackLeaks)("open", "close")
-], DataPipelineControllerImpl);
+], DataPipeline);
 var snapshotTimeframeToStartingTimeframe = (snapshotTimeframe) => {
   return snapshotTimeframe.map(([key, seq]) => [
     key,
@@ -1706,7 +1748,7 @@ var TestAgent = class {
     await metadataStore.addSpace({
       key: spaceKey
     });
-    const dataPipelineController = new DataPipelineControllerImpl({
+    const dataPipelineController = new DataPipeline({
       modelFactory: new import_model_factory.ModelFactory().registerModel(import_document_model.DocumentModel),
       metadataStore,
       snapshotManager,
@@ -1722,7 +1764,15 @@ var TestAgent = class {
       feedProvider: (feedKey) => this.feedStore.openFeed(feedKey)
     }).setControlFeed(controlFeed).setDataFeed(dataFeed);
     await space.open();
-    await space.initDataPipeline(dataPipelineController);
+    await dataPipelineController.open({
+      openPipeline: async (start) => {
+        const pipeline = await space.createDataPipeline({
+          start
+        });
+        await pipeline.start();
+        return pipeline;
+      }
+    });
     this._spaces.set(spaceKey, space);
     return [
       space,