@dxos/echo-pipeline 0.1.50 → 0.1.51-main.04cd027

package/dist/lib/browser/{chunk-CSFWTLPD.mjs → chunk-KTQITKUX.mjs}

@@ -121,6 +121,10 @@ var MetadataStore = class {
     }
   }
   _getSpace(spaceKey) {
+    var _a;
+    if ((_a = this._metadata.identity) == null ? void 0 : _a.haloSpace.key.equals(spaceKey)) {
+      return this._metadata.identity.haloSpace;
+    }
     const space = this.spaces.find((space2) => space2.key === spaceKey);
     assert(space, "Space not found");
     return space;
@@ -131,7 +135,7 @@ var MetadataStore = class {
   async clear() {
     log("clearing all metadata", {}, {
       file: "metadata-store.ts",
-      line: 131,
+      line: 136,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -152,10 +156,14 @@ var MetadataStore = class {
     ((_c = (_b = this._metadata).spaces) != null ? _c : _b.spaces = []).push(record);
     await this._save();
   }
-  async setSpaceLatestTimeframe(spaceKey, timeframe) {
+  async setSpaceDataLatestTimeframe(spaceKey, timeframe) {
     this._getSpace(spaceKey).dataTimeframe = timeframe;
     await this._save();
   }
+  async setSpaceControlLatestTimeframe(spaceKey, timeframe) {
+    this._getSpace(spaceKey).controlTimeframe = timeframe;
+    await this._save();
+  }
   async setCache(spaceKey, cache) {
     this._getSpace(spaceKey).cache = cache;
     await this._save();
@@ -334,6 +342,7 @@ var PipelineState = class {
     this._timeframeClock = _timeframeClock;
     this.timeframeUpdate = this._timeframeClock.update;
     this.stalled = new Event3();
+    this._startTimeframe = new Timeframe2();
   }
   /**
   * Latest theoretical timeframe based on the last mutation in each feed.
@@ -346,6 +355,9 @@ var PipelineState = class {
       index: feed.length - 1
     })));
   }
+  get startTimeframe() {
+    return this._startTimeframe;
+  }
   get timeframe() {
     return this._timeframeClock.timeframe;
   }
@@ -379,7 +391,7 @@ var PipelineState = class {
       target: this.targetTimeframe
     }, {
       file: "pipeline.ts",
-      line: 109,
+      line: 118,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -409,7 +421,7 @@ var PipelineState = class {
             dependencies: Timeframe2.dependencies(this.targetTimeframe, this.timeframe)
           }, {
             file: "pipeline.ts",
-            line: 135,
+            line: 144,
             scope: this,
             callSite: (f, a) => f(...a)
           });
@@ -466,7 +478,7 @@ var Pipeline = class {
   async start() {
     log4("starting...", {}, {
       file: "pipeline.ts",
-      line: 254,
+      line: 263,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -474,7 +486,7 @@ var Pipeline = class {
     await this._feedSetIterator.open();
     log4("started", {}, {
       file: "pipeline.ts",
-      line: 257,
+      line: 266,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -484,7 +496,7 @@ var Pipeline = class {
     var _a;
     log4("stopping...", {}, {
       file: "pipeline.ts",
-      line: 263,
+      line: 272,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -493,7 +505,7 @@ var Pipeline = class {
     await this._processingTrigger.wait();
     log4("stopped", {}, {
       file: "pipeline.ts",
-      line: 267,
+      line: 276,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -504,7 +516,35 @@ var Pipeline = class {
   */
   async setCursor(timeframe) {
     assert4(!this._isStarted || this._isPaused, "Invalid state.");
+    this._state._startTimeframe = timeframe;
     this._timeframeClock.setTimeframe(timeframe);
+    for (const [key, seq] of timeframe.frames()) {
+      const feed = this._feeds.get(key);
+      if (!feed) {
+        throw new Error("Feed not found");
+      }
+      feed.undownload({
+        callback: () => log4("Undownloaded", {}, {
+          file: "pipeline.ts",
+          line: 297,
+          scope: this,
+          callSite: (f, a) => f(...a)
+        })
+      });
+      feed.download({
+        start: seq + 1,
+        linear: true
+      }).catch((err) => {
+        log4("failed to download feed", {
+          err
+        }, {
+          file: "pipeline.ts",
+          line: 300,
+          scope: this,
+          callSite: (f, a) => f(...a)
+        });
+      });
+    }
     if (this._feedSetIterator) {
       await this._feedSetIterator.close();
       await this._initIterator();
@@ -567,7 +607,7 @@ var Pipeline = class {
     this._feedSetIterator.stalled.on((iterator) => {
       log4.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, {}, {
         file: "pipeline.ts",
-        line: 357,
+        line: 381,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -827,6 +867,7 @@ var DatabaseHost = class {
 
 // packages/core/echo/echo-pipeline/src/dbhost/snapshot-manager.ts
 import { trackLeaks } from "@dxos/async";
+import { cancelWithContext } from "@dxos/context";
 import { timed as timed2 } from "@dxos/debug";
 import { log as log7 } from "@dxos/log";
 import { schema as schema4 } from "@dxos/protocols";
@@ -853,7 +894,7 @@ var SnapshotManager = class SnapshotManager2 {
           snapshot
         }, {
           file: "snapshot-manager.ts",
-          line: 30,
+          line: 31,
           scope: this,
           callSite: (f, a) => f(...a)
         });
@@ -870,7 +911,7 @@ var SnapshotManager = class SnapshotManager2 {
           data
         }, {
           file: "snapshot-manager.ts",
-          line: 40,
+          line: 41,
           scope: this,
           callSite: (f, a) => f(...a)
         });
@@ -888,12 +929,12 @@ var SnapshotManager = class SnapshotManager2 {
   async close() {
     await this._objectSync.close();
   }
-  async load(id) {
-    const local = await this._snapshotStore.loadSnapshot(id);
+  async load(ctx, id) {
+    const local = await cancelWithContext(ctx, this._snapshotStore.loadSnapshot(id));
     if (local) {
       return local;
     }
-    const remote = await this._objectSync.download(id);
+    const remote = await cancelWithContext(ctx, this._objectSync.download(ctx, id));
     return schema4.getCodecForType("dxos.echo.snapshot.SpaceSnapshot").decode(remote.payload.value);
   }
   async store(snapshot) {
@@ -1017,6 +1058,7 @@ import { Event as Event4, scheduleTask as scheduleTask2, synchronized as synchro
 import { Context as Context3 } from "@dxos/context";
 import { checkCredentialType } from "@dxos/credentials";
 import { getStateMachineFromItem, ItemManager } from "@dxos/echo-db";
+import { CancelledError } from "@dxos/errors";
 import { log as log9 } from "@dxos/log";
 import { Timeframe as Timeframe3 } from "@dxos/timeframe";
 var __decorate5 = function(decorators, target, key, desc) {
@@ -1040,6 +1082,7 @@ var DataPipeline = class DataPipeline2 {
     this._isOpen = false;
     this._lastTimeframeSaveTime = 0;
     this._lastSnapshotSaveTime = 0;
+    this._lastProcessedEpoch = -1;
     this.onNewEpoch = new Event4();
   }
   get isOpen() {
@@ -1063,17 +1106,10 @@ var DataPipeline = class DataPipeline2 {
         if (!checkCredentialType(credential, "dxos.halo.credentials.Epoch")) {
           return;
         }
-        log9("new epoch", {
-          credential
-        }, {
-          file: "data-pipeline.ts",
-          line: 109,
-          scope: this,
-          callSite: (f, a) => f(...a)
-        });
-        await this._processEpoch(credential.subject.assertion);
         this.currentEpoch = credential;
-        this.onNewEpoch.emit(credential);
+        if (this._isOpen) {
+          await this._processEpochInSeparateTask(credential);
+        }
       }
     };
   }
@@ -1112,7 +1148,7 @@ var DataPipeline = class DataPipeline2 {
     }
     log9("close", {}, {
       file: "data-pipeline.ts",
-      line: 161,
+      line: 174,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -1127,7 +1163,7 @@ var DataPipeline = class DataPipeline2 {
     } catch (err) {
       log9.catch(err, {}, {
         file: "data-pipeline.ts",
-        line: 174,
+        line: 187,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -1137,6 +1173,11 @@ var DataPipeline = class DataPipeline2 {
     await this._params.snapshotManager.close();
   }
   async _consumePipeline() {
+    if (this.currentEpoch) {
+      const waitForOneEpoch = this.onNewEpoch.waitForCount(1);
+      await this._processEpochInSeparateTask(this.currentEpoch);
+      await waitForOneEpoch;
+    }
     assert8(this._pipeline, "Pipeline is not initialized.");
     for await (const msg of this._pipeline.consume()) {
       const { feedKey, seq, data } = msg;
@@ -1145,7 +1186,7 @@ var DataPipeline = class DataPipeline2 {
         seq
       }, {
         file: "data-pipeline.ts",
-        line: 186,
+        line: 205,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -1157,7 +1198,7 @@ var DataPipeline = class DataPipeline2 {
               feedKey
             }, {
               file: "data-pipeline.ts",
-              line: 192,
+              line: 211,
               scope: this,
               callSite: (f, a) => f(...a)
             });
@@ -1172,12 +1213,22 @@ var DataPipeline = class DataPipeline2 {
               memberKey: feedInfo.assertion.identityKey
             }
           });
+          log9.trace("dxos.echo.data-pipeline.processed", {
+            feedKey: feedKey.toHex(),
+            seq,
+            spaceKey: this._params.spaceKey.toHex()
+          }, {
+            file: "data-pipeline.ts",
+            line: 225,
+            scope: this,
+            callSite: (f, a) => f(...a)
+          });
           await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
         }
       } catch (err) {
         log9.catch(err, {}, {
           file: "data-pipeline.ts",
-          line: 210,
+          line: 235,
           scope: this,
           callSite: (f, a) => f(...a)
         });
@@ -1195,7 +1246,7 @@ var DataPipeline = class DataPipeline2 {
   async _saveTargetTimeframe(timeframe) {
     var _a;
     const newTimeframe = Timeframe3.merge((_a = this._targetTimeframe) != null ? _a : new Timeframe3(), timeframe);
-    await this._params.metadataStore.setSpaceLatestTimeframe(this._params.spaceKey, newTimeframe);
+    await this._params.metadataStore.setSpaceDataLatestTimeframe(this._params.spaceKey, newTimeframe);
     this._targetTimeframe = newTimeframe;
   }
   async _saveCache() {
@@ -1212,7 +1263,7 @@ var DataPipeline = class DataPipeline2 {
     } catch (err) {
       log9.warn("Failed to cache properties", err, {
         file: "data-pipeline.ts",
-        line: 244,
+        line: 269,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -1228,24 +1279,68 @@ var DataPipeline = class DataPipeline2 {
       await this._saveCache();
     }
   }
-  async _processEpoch(epoch) {
-    assert8(this._isOpen);
+  async _processEpochInSeparateTask(epoch) {
+    var _a;
+    if (epoch.subject.assertion.number <= this._lastProcessedEpoch) {
+      return;
+    }
+    await ((_a = this._epochCtx) == null ? void 0 : _a.dispose());
+    const ctx = new Context3({
+      onError: (err) => {
+        if (err instanceof CancelledError) {
+          log9("Epoch processing cancelled.", {}, {
+            file: "data-pipeline.ts",
+            line: 301,
+            scope: this,
+            callSite: (f, a) => f(...a)
+          });
+        } else {
+          log9.catch(err, {}, {
+            file: "data-pipeline.ts",
+            line: 303,
+            scope: this,
+            callSite: (f, a) => f(...a)
+          });
+        }
+      }
+    });
+    this._epochCtx = ctx;
+    scheduleTask2(ctx, async () => {
+      if (!this._isOpen) {
+        return;
+      }
+      log9("process epoch", {
+        epoch
+      }, {
+        file: "data-pipeline.ts",
+        line: 313,
+        scope: this,
+        callSite: (f, a) => f(...a)
+      });
+      await this._processEpoch(ctx, epoch.subject.assertion);
+      this.appliedEpoch = epoch;
+      this.onNewEpoch.emit(epoch);
+    });
+  }
+  async _processEpoch(ctx, epoch) {
+    assert8(this._isOpen, "Space is closed.");
     assert8(this._pipeline);
+    this._lastProcessedEpoch = epoch.number;
     log9("Processing epoch", {
       epoch
     }, {
       file: "data-pipeline.ts",
-      line: 273,
+      line: 327,
       scope: this,
       callSite: (f, a) => f(...a)
     });
     if (epoch.snapshotCid) {
-      const snapshot = await this._params.snapshotManager.load(epoch.snapshotCid);
+      const snapshot = await this._params.snapshotManager.load(ctx, epoch.snapshotCid);
       this.databaseHost._itemDemuxer.restoreFromSnapshot(snapshot.database);
     }
     log9("restarting pipeline for epoch", {}, {
       file: "data-pipeline.ts",
-      line: 281,
+      line: 335,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -1304,10 +1399,15 @@ import { SpaceStateMachine } from "@dxos/credentials";
 import { PublicKey as PublicKey3 } from "@dxos/keys";
 import { log as log10 } from "@dxos/log";
 import { AdmittedFeed } from "@dxos/protocols/proto/dxos/halo/credentials";
+import { Timeframe as Timeframe4 } from "@dxos/timeframe";
 import { Callback } from "@dxos/util";
+var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL2 = 500;
 var ControlPipeline = class {
-  constructor({ spaceKey, genesisFeed, feedProvider }) {
+  constructor({ spaceKey, genesisFeed, feedProvider, metadataStore }) {
+    this._lastTimeframeSaveTime = Date.now();
     this.onFeedAdmitted = new Callback();
+    this._spaceKey = spaceKey;
+    this._metadata = metadataStore;
     this._pipeline = new Pipeline();
     void this._pipeline.addFeed(genesisFeed);
     this._spaceStateMachine = new SpaceStateMachine(spaceKey);
@@ -1316,7 +1416,7 @@ var ControlPipeline = class {
         key: info.key
       }, {
         file: "control-pipeline.ts",
-        line: 39,
+        line: 51,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -1327,7 +1427,7 @@ var ControlPipeline = class {
         } catch (err) {
           log10.catch(err, {}, {
             file: "control-pipeline.ts",
-            line: 47,
+            line: 59,
             scope: this,
             callSite: (f, a) => f(...a)
           });
@@ -1350,7 +1450,7 @@ var ControlPipeline = class {
   async start() {
     log10("starting...", {}, {
       file: "control-pipeline.ts",
-      line: 71,
+      line: 83,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -1362,7 +1462,7 @@ var ControlPipeline = class {
             seq: msg.seq
           }, {
             file: "control-pipeline.ts",
-            line: 76,
+            line: 88,
             scope: this,
             callSite: (f, a) => f(...a)
           });
@@ -1373,16 +1473,18 @@ var ControlPipeline = class {
                 msg
               }, {
                 file: "control-pipeline.ts",
-                line: 83,
+                line: 95,
                 scope: this,
                 callSite: (f, a) => f(...a)
               });
+            } else {
+              await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
             }
           }
         } catch (err) {
           log10.catch(err, {}, {
             file: "control-pipeline.ts",
-            line: 87,
+            line: 101,
             scope: this,
             callSite: (f, a) => f(...a)
           });
@@ -1392,26 +1494,48 @@ var ControlPipeline = class {
     await this._pipeline.start();
     log10("started", {}, {
       file: "control-pipeline.ts",
-      line: 93,
+      line: 107,
       scope: this,
       callSite: (f, a) => f(...a)
     });
   }
+  async _noteTargetStateIfNeeded(timeframe) {
+    if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL2) {
+      this._lastTimeframeSaveTime = Date.now();
+      await this._saveTargetTimeframe(timeframe);
+    }
+  }
   async stop() {
     log10("stopping...", {}, {
       file: "control-pipeline.ts",
-      line: 97,
+      line: 121,
       scope: this,
       callSite: (f, a) => f(...a)
     });
     await this._pipeline.stop();
+    await this._saveTargetTimeframe(this._pipeline.state.timeframe);
     log10("stopped", {}, {
       file: "control-pipeline.ts",
-      line: 99,
+      line: 124,
       scope: this,
       callSite: (f, a) => f(...a)
     });
   }
+  async _saveTargetTimeframe(timeframe) {
+    var _a;
+    try {
+      const newTimeframe = Timeframe4.merge((_a = this._targetTimeframe) != null ? _a : new Timeframe4(), timeframe);
+      await this._metadata.setSpaceControlLatestTimeframe(this._spaceKey, newTimeframe);
+      this._targetTimeframe = newTimeframe;
+    } catch (err) {
+      log10(err, {}, {
+        file: "control-pipeline.ts",
+        line: 133,
+        scope: this,
+        callSite: (f, a) => f(...a)
+      });
+    }
+  }
 };
 
 // packages/core/echo/echo-pipeline/src/space/space.ts
@@ -1441,20 +1565,26 @@ var Space = class Space2 {
     this._controlPipeline = new ControlPipeline({
       spaceKey: params.spaceKey,
       genesisFeed: params.genesisFeed,
-      feedProvider: params.feedProvider
+      feedProvider: params.feedProvider,
+      metadataStore: params.metadataStore
     });
     this._controlPipeline.onFeedAdmitted.set(async (info) => {
+      const sparse = info.assertion.designation === AdmittedFeed2.Designation.DATA;
       if (info.assertion.designation === AdmittedFeed2.Designation.DATA) {
         await this._addFeedLock.executeSynchronized(async () => {
           if (this._dataPipeline.pipeline) {
             if (!this._dataPipeline.pipeline.hasFeed(info.key)) {
-              return this._dataPipeline.pipeline.addFeed(await this._feedProvider(info.key));
+              return this._dataPipeline.pipeline.addFeed(await this._feedProvider(info.key, {
+                sparse
+              }));
             }
           }
         });
       }
       if (!info.key.equals(params.genesisFeed.key)) {
-        this.protocol.addFeed(await params.feedProvider(info.key));
+        this.protocol.addFeed(await params.feedProvider(info.key, {
+          sparse
+        }));
       }
     });
     this._controlPipeline.onCredentialProcessed.set(async (credential) => {
@@ -1463,7 +1593,7 @@ var Space = class Space2 {
         credential
       }, {
         file: "space.ts",
-        line: 106,
+        line: 110,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -1486,7 +1616,9 @@ var Space = class Space2 {
         await this._addFeedLock.executeSynchronized(async () => {
           for (const feed of this._controlPipeline.spaceState.feeds.values()) {
             if (feed.assertion.designation === AdmittedFeed2.Designation.DATA && !pipeline.hasFeed(feed.key)) {
-              await pipeline.addFeed(await this._feedProvider(feed.key));
+              await pipeline.addFeed(await this._feedProvider(feed.key, {
+                sparse: true
+              }));
             }
           }
         });
@@ -1553,7 +1685,7 @@ var Space = class Space2 {
   async open() {
     log11("opening...", {}, {
       file: "space.ts",
-      line: 208,
+      line: 212,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -1563,10 +1695,11 @@ var Space = class Space2 {
     await this._controlPipeline.start();
     await this.protocol.start();
     this._dataPipelineCredentialConsumer = this._controlPipeline.spaceState.registerProcessor(this._dataPipeline.createCredentialProcessor());
+    await this._dataPipelineCredentialConsumer.open();
     this._isOpen = true;
     log11("opened", {}, {
       file: "space.ts",
-      line: 222,
+      line: 228,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -1577,7 +1710,7 @@ var Space = class Space2 {
       key: this._key
     }, {
       file: "space.ts",
-      line: 227,
+      line: 233,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -1592,7 +1725,7 @@ var Space = class Space2 {
     this._isOpen = false;
     log11("closed", {}, {
       file: "space.ts",
-      line: 241,
+      line: 247,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -1600,14 +1733,12 @@ var Space = class Space2 {
   async initializeDataPipeline() {
     log11("initializeDataPipeline", {}, {
       file: "space.ts",
-      line: 246,
+      line: 252,
       scope: this,
       callSite: (f, a) => f(...a)
     });
     assert9(this._isOpen, "Space must be open to initialize data pipeline.");
     await this._dataPipeline.open();
-    assert9(this._dataPipelineCredentialConsumer);
-    await this._dataPipelineCredentialConsumer.open();
   }
 };
 __decorate6([
@@ -1880,7 +2011,7 @@ var SpaceManager = class SpaceManager2 {
       spaceKey,
       protocol,
       genesisFeed,
-      feedProvider: (feedKey) => this._feedStore.openFeed(feedKey),
+      feedProvider: (feedKey, opts) => this._feedStore.openFeed(feedKey, opts),
       modelFactory: this._modelFactory,
       metadataStore: this._metadataStore,
       snapshotManager,
@@ -1934,4 +2065,4 @@ export {
   SpaceProtocolSession,
   SpaceManager
 };
-//# sourceMappingURL=chunk-CSFWTLPD.mjs.map
+//# sourceMappingURL=chunk-KTQITKUX.mjs.map