@dxos/echo-pipeline 0.1.36 → 0.1.37-next.8c6108a

package/dist/lib/browser/{chunk-4YAT2XJW.mjs → chunk-4TOOIHMA.mjs}

@@ -1,10 +1,8 @@
 import "@dxos/node-std/globals"
 
 // packages/core/echo/echo-pipeline/src/metadata/metadata-store.ts
-import CRC32 from "crc-32";
 import assert from "@dxos/node-std/assert";
 import { synchronized } from "@dxos/async";
-import { DataCorruptionError } from "@dxos/errors";
 import { log } from "@dxos/log";
 import { schema } from "@dxos/protocols";
 var __decorate = function(decorators, target, key, desc) {
@@ -49,35 +47,31 @@ var MetadataStore = class {
     const file = this._directory.getOrCreateFile("EchoMetadata");
     try {
       const { size: fileLength } = await file.stat();
-      if (fileLength < 8) {
+      if (fileLength < 4) {
         return;
       }
       const dataSize = fromBytesInt32(await file.read(0, 4));
-      const checksum = fromBytesInt32(await file.read(4, 4));
       log("loaded", {
-        size: dataSize,
-        checksum
+        size: dataSize
       }, {
         file: "metadata-store.ts",
-        line: 70,
+        line: 67,
         scope: this,
         callSite: (f, a) => f(...a)
       });
-      if (fileLength < dataSize + 8) {
-        throw new DataCorruptionError("Metadata size is smaller than expected.");
-      }
-      const data = await file.read(8, dataSize);
-      const calculatedChecksum = CRC32.buf(data);
-      if (calculatedChecksum !== checksum) {
-        throw new DataCorruptionError("Metadata checksum is invalid.");
+      {
+        if (fileLength < dataSize + 4) {
+          throw new Error("Metadata storage is corrupted");
+        }
       }
+      const data = await file.read(4, dataSize);
       this._metadata = schema.getCodecForType("dxos.echo.metadata.EchoMetadata").decode(data);
     } catch (err) {
       log.error("failed to load metadata", {
         err
       }, {
         file: "metadata-store.ts",
-        line: 85,
+        line: 79,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -97,21 +91,16 @@ var MetadataStore = class {
     const file = this._directory.getOrCreateFile("EchoMetadata");
     try {
       const encoded = Buffer.from(schema.getCodecForType("dxos.echo.metadata.EchoMetadata").encode(data));
-      const checksum = CRC32.buf(encoded);
-      const result = Buffer.alloc(8 + encoded.length);
-      result.writeInt32LE(encoded.length, 0);
-      result.writeInt32LE(checksum, 4);
-      encoded.copy(result, 8);
-      await file.write(0, result);
+      await file.write(0, toBytesInt32(encoded.length));
       log("saved", {
-        size: encoded.length,
-        checksum
+        size: encoded.length
       }, {
         file: "metadata-store.ts",
-        line: 116,
+        line: 102,
         scope: this,
         callSite: (f, a) => f(...a)
       });
+      await file.write(4, encoded);
     } finally {
       await file.close();
     }
@@ -122,7 +111,7 @@ var MetadataStore = class {
   async clear() {
     log("clearing all metadata", {}, {
       file: "metadata-store.ts",
-      line: 126,
+      line: 115,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -168,6 +157,11 @@ __decorate([
 __decorate([
   synchronized
 ], MetadataStore.prototype, "_save", null);
+var toBytesInt32 = (num) => {
+  const buf = Buffer.alloc(4);
+  buf.writeInt32LE(num, 0);
+  return buf;
+};
 var fromBytesInt32 = (buf) => buf.readInt32LE(0);
 
 // packages/core/echo/echo-pipeline/src/common/codec.ts
@@ -218,31 +212,17 @@ var TimeframeClock = class {
   constructor(_timeframe = new Timeframe()) {
     this._timeframe = _timeframe;
     this.update = new Event();
-    this._pendingTimeframe = _timeframe;
   }
-  /**
-  * Timeframe that was processed by ECHO.
-  */
   get timeframe() {
     return this._timeframe;
   }
-  /**
-  * Timeframe that is currently being processed by ECHO.
-  * Will be equal to `timeframe` after the processing is complete.
-  */
-  get pendingTimeframe() {
-    return this._pendingTimeframe;
-  }
-  updatePendingTimeframe(key, seq) {
-    this._pendingTimeframe = Timeframe.merge(this._pendingTimeframe, new Timeframe([
+  updateTimeframe(key, seq) {
+    this._timeframe = Timeframe.merge(this._timeframe, new Timeframe([
       [
         key,
         seq
       ]
     ]));
-  }
-  updateTimeframe() {
-    this._timeframe = this._pendingTimeframe;
     this.update.emit(this._timeframe);
   }
   hasGaps(timeframe) {
@@ -255,7 +235,7 @@ var TimeframeClock = class {
       current: this._timeframe
     }, {
       file: "timeframe-clock.ts",
-      line: 67,
+      line: 48,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -266,7 +246,7 @@ var TimeframeClock = class {
         deps: Timeframe.dependencies(target, this._timeframe)
       }, {
         file: "timeframe-clock.ts",
-        line: 69,
+        line: 50,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -336,9 +316,6 @@ var PipelineState = class {
   get timeframe() {
     return this._timeframeClock.timeframe;
   }
-  get pendingTimeframe() {
-    return this._timeframeClock.pendingTimeframe;
-  }
   get targetTimeframe() {
     return this._targetTimeframe ? this._targetTimeframe : new Timeframe2();
   }
@@ -366,7 +343,7 @@ var PipelineState = class {
       target: this.targetTimeframe
     }, {
       file: "pipeline.ts",
-      line: 105,
+      line: 101,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -396,7 +373,7 @@ var PipelineState = class {
             dependencies: Timeframe2.dependencies(this.targetTimeframe, this.timeframe)
           }, {
             file: "pipeline.ts",
-            line: 131,
+            line: 127,
             scope: this,
             callSite: (f, a) => f(...a)
           });
@@ -422,7 +399,7 @@ var Pipeline = class {
     this._feedSetIterator.stalled.on((iterator) => {
       log4.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, {}, {
         file: "pipeline.ts",
-        line: 207,
+        line: 203,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -457,14 +434,14 @@ var Pipeline = class {
   async start() {
     log4("starting...", {}, {
       file: "pipeline.ts",
-      line: 249,
+      line: 245,
       scope: this,
       callSite: (f, a) => f(...a)
     });
     await this._feedSetIterator.open();
     log4("started", {}, {
       file: "pipeline.ts",
-      line: 251,
+      line: 247,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -472,7 +449,7 @@ var Pipeline = class {
   async stop() {
     log4("stopping...", {}, {
       file: "pipeline.ts",
-      line: 256,
+      line: 252,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -480,7 +457,7 @@ var Pipeline = class {
     await this._processingTrigger.wait();
     log4("stopped", {}, {
       file: "pipeline.ts",
-      line: 259,
+      line: 255,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -494,10 +471,9 @@ var Pipeline = class {
     this._isOpen = true;
     for await (const block of this._feedSetIterator) {
       this._processingTrigger.reset();
-      this._timeframeClock.updatePendingTimeframe(PublicKey.from(block.feedKey), block.seq);
       yield block;
       this._processingTrigger.wake();
-      this._timeframeClock.updateTimeframe();
+      this._timeframeClock.updateTimeframe(PublicKey.from(block.feedKey), block.seq);
     }
     this._isOpen = false;
   }
@@ -1530,7 +1506,7 @@ var DataServiceImpl = class {
 
 // packages/core/echo/echo-pipeline/src/space/data-pipeline.ts
 import assert9 from "@dxos/node-std/assert";
-import { scheduleTask as scheduleTask2, synchronized as synchronized5, trackLeaks as trackLeaks4 } from "@dxos/async";
+import { Event as Event3, scheduleTask as scheduleTask2, synchronized as synchronized5, trackLeaks as trackLeaks4 } from "@dxos/async";
 import { Context as Context3 } from "@dxos/context";
 import { failUndefined as failUndefined3 } from "@dxos/debug";
 import { ItemManager } from "@dxos/echo-db";
@@ -1556,8 +1532,7 @@ var DataPipeline = class DataPipeline2 {
     this._ctx = new Context3();
     this._lastAutomaticSnapshotTimeframe = new Timeframe3();
     this._isOpen = false;
-    this._lastTimeframeSaveTime = 0;
-    this._lastSnapshotSaveTime = 0;
+    this.onTimeframeReached = new Event3();
   }
   get isOpen() {
     return this._isOpen;
@@ -1602,8 +1577,58 @@ var DataPipeline = class DataPipeline2 {
     scheduleTask2(this._ctx, async () => {
       await this._consumePipeline();
     });
+    this._createPeriodicSnapshots();
     this._isOpen = true;
   }
+  _createPeriodicSnapshots() {
+    this.onTimeframeReached.debounce(TIMEFRAME_SAVE_DEBOUNCE_INTERVAL).on(this._ctx, async () => {
+      await this._saveLatestTimeframe();
+    });
+    if (!DISABLE_SNAPSHOT_CACHE) {
+      this.onTimeframeReached.debounce(AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL).on(this._ctx, async () => {
+        var _a, _b;
+        const latestTimeframe = (_a = this._pipeline) == null ? void 0 : _a.state.timeframe;
+        if (!latestTimeframe) {
+          return;
+        }
+        if (latestTimeframe.totalMessages() - this._lastAutomaticSnapshotTimeframe.totalMessages() > MESSAGES_PER_SNAPSHOT) {
+          const snapshot = await this._saveSnapshot();
+          this._lastAutomaticSnapshotTimeframe = (_b = snapshot.timeframe) != null ? _b : failUndefined3();
+          log12("save", {
+            snapshot
+          }, {
+            file: "data-pipeline.ts",
+            line: 161,
+            scope: this,
+            callSite: (f, a) => f(...a)
+          });
+        }
+      });
+    }
+  }
+  async _saveSnapshot() {
+    const snapshot = await this.createSnapshot();
+    const snapshotKey = await this._params.snapshotManager.store(snapshot);
+    await this._params.metadataStore.setSpaceSnapshot(this._params.spaceKey, snapshotKey);
+    return snapshot;
+  }
+  async _saveLatestTimeframe() {
+    var _a, _b;
+    const latestTimeframe = (_a = this._pipeline) == null ? void 0 : _a.state.timeframe;
+    log12("save latest timeframe", {
+      latestTimeframe,
+      spaceKey: this._params.spaceKey
+    }, {
+      file: "data-pipeline.ts",
+      line: 176,
+      scope: this,
+      callSite: (f, a) => f(...a)
+    });
+    if (latestTimeframe) {
+      const newTimeframe = Timeframe3.merge((_b = this._targetTimeframe) != null ? _b : new Timeframe3(), latestTimeframe);
+      await this._params.metadataStore.setSpaceLatestTimeframe(this._params.spaceKey, newTimeframe);
+    }
+  }
   async close() {
     var _a, _b, _c;
     if (!this._isOpen) {
@@ -1611,32 +1636,37 @@ var DataPipeline = class DataPipeline2 {
     }
     log12("close", {}, {
       file: "data-pipeline.ts",
-      line: 145,
+      line: 188,
       scope: this,
       callSite: (f, a) => f(...a)
     });
     this._isOpen = false;
-    await this._ctx.dispose();
-    await ((_a = this._pipeline) == null ? void 0 : _a.stop());
     try {
-      if (this._pipeline) {
-        await this._saveTargetTimeframe(this._pipeline.state.timeframe);
-        if (!DISABLE_SNAPSHOT_CACHE) {
-          await this._saveSnapshot(this._pipeline.state.timeframe);
-        }
-      }
+      await this._saveLatestTimeframe();
+      await this._saveSnapshot();
     } catch (err) {
       log12.catch(err, {}, {
         file: "data-pipeline.ts",
-        line: 160,
+        line: 195,
         scope: this,
         callSite: (f, a) => f(...a)
       });
     }
+    await this._ctx.dispose();
+    await ((_a = this._pipeline) == null ? void 0 : _a.stop());
     await ((_b = this.databaseBackend) == null ? void 0 : _b.close());
     await ((_c = this._itemManager) == null ? void 0 : _c.destroy());
     await this._params.snapshotManager.close();
   }
+  createSnapshot() {
+    var _a, _b;
+    assert9(this.databaseBackend, "Database backend is not initialized.");
+    return {
+      spaceKey: this._params.spaceKey.asUint8Array(),
+      timeframe: (_b = (_a = this._pipeline) == null ? void 0 : _a.state.timeframe) != null ? _b : new Timeframe3(),
+      database: this.databaseBackend.createSnapshot()
+    };
+  }
   async _consumePipeline() {
     assert9(this._pipeline, "Pipeline is not initialized.");
     for await (const msg of this._pipeline.consume()) {
@@ -1645,7 +1675,7 @@ var DataPipeline = class DataPipeline2 {
         msg
       }, {
         file: "data-pipeline.ts",
-        line: 172,
+        line: 217,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -1657,7 +1687,7 @@ var DataPipeline = class DataPipeline2 {
               feedKey
             }, {
               file: "data-pipeline.ts",
-              line: 178,
+              line: 223,
               scope: this,
               callSite: (f, a) => f(...a)
             });
@@ -1672,58 +1702,18 @@ var DataPipeline = class DataPipeline2 {
               memberKey: feedInfo.assertion.identityKey
             }
           });
-          await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
+          this.onTimeframeReached.emit(data.timeframe);
         }
       } catch (err) {
         log12.catch(err, {}, {
           file: "data-pipeline.ts",
-          line: 196,
+          line: 239,
           scope: this,
           callSite: (f, a) => f(...a)
         });
       }
     }
   }
-  _createSnapshot(timeframe) {
-    assert9(this.databaseBackend, "Database backend is not initialized.");
-    return {
-      spaceKey: this._params.spaceKey.asUint8Array(),
-      timeframe,
-      database: this.databaseBackend.createSnapshot()
-    };
-  }
-  async _saveSnapshot(timeframe) {
-    const snapshot = await this._createSnapshot(timeframe);
-    const snapshotKey = await this._params.snapshotManager.store(snapshot);
-    await this._params.metadataStore.setSpaceSnapshot(this._params.spaceKey, snapshotKey);
-    return snapshot;
-  }
-  async _saveTargetTimeframe(timeframe) {
-    var _a;
-    const newTimeframe = Timeframe3.merge((_a = this._targetTimeframe) != null ? _a : new Timeframe3(), timeframe);
-    await this._params.metadataStore.setSpaceLatestTimeframe(this._params.spaceKey, newTimeframe);
-    this._targetTimeframe = newTimeframe;
-  }
-  async _noteTargetStateIfNeeded(timeframe) {
-    var _a;
-    if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL) {
-      this._lastTimeframeSaveTime = Date.now();
-      await this._saveTargetTimeframe(timeframe);
-    }
-    if (!DISABLE_SNAPSHOT_CACHE && Date.now() - this._lastSnapshotSaveTime > AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL && timeframe.totalMessages() - this._lastAutomaticSnapshotTimeframe.totalMessages() > MESSAGES_PER_SNAPSHOT) {
-      this._lastSnapshotSaveTime = Date.now();
-      const snapshot = await this._saveSnapshot(timeframe);
-      this._lastAutomaticSnapshotTimeframe = (_a = snapshot.timeframe) != null ? _a : failUndefined3();
-      log12("save", {
-        snapshot
-      }, {
-        file: "data-pipeline.ts",
-        line: 239,
-        scope: this,
-        callSite: (f, a) => f(...a)
-      });
-    }
-  }
   async waitUntilTimeframe(timeframe) {
     assert9(this._pipeline, "Pipeline is not initialized.");
     await this._pipeline.state.waitUntilTimeframe(timeframe);
@@ -1773,4 +1763,4 @@ export {
   DataServiceImpl,
   DataPipeline
 };
-//# sourceMappingURL=chunk-4YAT2XJW.mjs.map
+//# sourceMappingURL=chunk-4TOOIHMA.mjs.map