@dxos/echo-pipeline 0.1.50 → 0.1.51-main.04cd027

package/dist/lib/node/testing/index.cjs

@@ -200,6 +200,7 @@ var DatabaseHost = class {
 
 // packages/core/echo/echo-pipeline/src/dbhost/snapshot-manager.ts
 var import_async = require("@dxos/async");
+var import_context = require("@dxos/context");
 var import_debug = require("@dxos/debug");
 var import_log2 = require("@dxos/log");
 var import_protocols = require("@dxos/protocols");
@@ -226,7 +227,7 @@ var SnapshotManager = class SnapshotManager2 {
           snapshot
         }, {
           file: "snapshot-manager.ts",
-          line: 30,
+          line: 31,
           scope: this,
           callSite: (f, a) => f(...a)
         });
@@ -243,7 +244,7 @@ var SnapshotManager = class SnapshotManager2 {
           data
         }, {
           file: "snapshot-manager.ts",
-          line: 40,
+          line: 41,
           scope: this,
           callSite: (f, a) => f(...a)
         });
@@ -261,12 +262,12 @@ var SnapshotManager = class SnapshotManager2 {
   async close() {
     await this._objectSync.close();
   }
-  async load(id) {
-    const local = await this._snapshotStore.loadSnapshot(id);
+  async load(ctx, id) {
+    const local = await (0, import_context.cancelWithContext)(ctx, this._snapshotStore.loadSnapshot(id));
     if (local) {
       return local;
     }
-    const remote = await this._objectSync.download(id);
+    const remote = await (0, import_context.cancelWithContext)(ctx, this._objectSync.download(ctx, id));
     return import_protocols.schema.getCodecForType("dxos.echo.snapshot.SpaceSnapshot").decode(remote.payload.value);
   }
   async store(snapshot) {
@@ -505,6 +506,10 @@ var MetadataStore = class {
     }
   }
   _getSpace(spaceKey) {
+    var _a;
+    if ((_a = this._metadata.identity) == null ? void 0 : _a.haloSpace.key.equals(spaceKey)) {
+      return this._metadata.identity.haloSpace;
+    }
     const space = this.spaces.find((space2) => space2.key === spaceKey);
     (0, import_node_assert3.default)(space, "Space not found");
     return space;
@@ -515,7 +520,7 @@ var MetadataStore = class {
   async clear() {
     (0, import_log4.log)("clearing all metadata", {}, {
       file: "metadata-store.ts",
-      line: 131,
+      line: 136,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -536,10 +541,14 @@ var MetadataStore = class {
     ((_c = (_b = this._metadata).spaces) != null ? _c : _b.spaces = []).push(record);
     await this._save();
   }
-  async setSpaceLatestTimeframe(spaceKey, timeframe) {
+  async setSpaceDataLatestTimeframe(spaceKey, timeframe) {
     this._getSpace(spaceKey).dataTimeframe = timeframe;
     await this._save();
   }
+  async setSpaceControlLatestTimeframe(spaceKey, timeframe) {
+    this._getSpace(spaceKey).controlTimeframe = timeframe;
+    await this._save();
+  }
   async setCache(spaceKey, cache) {
     this._getSpace(spaceKey).cache = cache;
     await this._save();
@@ -562,7 +571,7 @@ var fromBytesInt32 = (buf) => buf.readInt32LE(0);
 // packages/core/echo/echo-pipeline/src/space/auth.ts
 var import_node_assert4 = __toESM(require("node:assert"));
 var import_async3 = require("@dxos/async");
-var import_context = require("@dxos/context");
+var import_context2 = require("@dxos/context");
 var import_crypto2 = require("@dxos/crypto");
 var import_log5 = require("@dxos/log");
 var import_protocols4 = require("@dxos/protocols");
@@ -579,7 +588,7 @@ var AuthExtension = class extends import_teleport.RpcExtension {
       timeout: 60 * 1e3
     });
     this._authParams = _authParams;
-    this._ctx = new import_context.Context({
+    this._ctx = new import_context2.Context({
       onError: (err) => {
         import_log5.log.catch(err, {}, {
           file: "auth.ts",
@@ -657,12 +666,13 @@ var import_credentials = require("@dxos/credentials");
 var import_keys3 = require("@dxos/keys");
 var import_log8 = require("@dxos/log");
 var import_credentials2 = require("@dxos/protocols/proto/dxos/halo/credentials");
+var import_timeframe3 = require("@dxos/timeframe");
 var import_util4 = require("@dxos/util");
 
 // packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts
 var import_node_assert7 = __toESM(require("node:assert"));
 var import_async5 = require("@dxos/async");
-var import_context2 = require("@dxos/context");
+var import_context3 = require("@dxos/context");
 var import_debug5 = require("@dxos/debug");
 var import_feed_store = require("@dxos/feed-store");
 var import_keys2 = require("@dxos/keys");
@@ -814,6 +824,7 @@ var PipelineState = class {
     this._timeframeClock = _timeframeClock;
     this.timeframeUpdate = this._timeframeClock.update;
     this.stalled = new import_async5.Event();
+    this._startTimeframe = new import_timeframe2.Timeframe();
   }
   /**
   * Latest theoretical timeframe based on the last mutation in each feed.
@@ -826,6 +837,9 @@ var PipelineState = class {
       index: feed.length - 1
     })));
   }
+  get startTimeframe() {
+    return this._startTimeframe;
+  }
   get timeframe() {
     return this._timeframeClock.timeframe;
   }
@@ -851,7 +865,7 @@ var PipelineState = class {
   *
   * @param timeout Timeout in milliseconds to specify the maximum wait time.
   */
-  async waitUntilReachedTargetTimeframe({ ctx = new import_context2.Context(), timeout, breakOnStall = true } = {}) {
+  async waitUntilReachedTargetTimeframe({ ctx = new import_context3.Context(), timeout, breakOnStall = true } = {}) {
     var _a;
     (0, import_log7.log)("waitUntilReachedTargetTimeframe", {
       timeout,
@@ -859,7 +873,7 @@ var PipelineState = class {
       target: this.targetTimeframe
     }, {
       file: "pipeline.ts",
-      line: 109,
+      line: 118,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -874,7 +888,7 @@ var PipelineState = class {
     let done = false;
     if (timeout) {
       return Promise.race([
-        (0, import_context2.rejectOnDispose)(ctx),
+        (0, import_context3.rejectOnDispose)(ctx),
         this._reachedTargetPromise.then(() => {
           done = true;
         }),
@@ -889,7 +903,7 @@ var PipelineState = class {
             dependencies: import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe)
           }, {
             file: "pipeline.ts",
-            line: 135,
+            line: 144,
             scope: this,
             callSite: (f, a) => f(...a)
           });
@@ -946,7 +960,7 @@ var Pipeline = class {
   async start() {
     (0, import_log7.log)("starting...", {}, {
       file: "pipeline.ts",
-      line: 254,
+      line: 263,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -954,7 +968,7 @@ var Pipeline = class {
     await this._feedSetIterator.open();
     (0, import_log7.log)("started", {}, {
       file: "pipeline.ts",
-      line: 257,
+      line: 266,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -964,7 +978,7 @@ var Pipeline = class {
     var _a;
     (0, import_log7.log)("stopping...", {}, {
       file: "pipeline.ts",
-      line: 263,
+      line: 272,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -973,7 +987,7 @@ var Pipeline = class {
     await this._processingTrigger.wait();
     (0, import_log7.log)("stopped", {}, {
       file: "pipeline.ts",
-      line: 267,
+      line: 276,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -984,7 +998,35 @@ var Pipeline = class {
   */
   async setCursor(timeframe) {
     (0, import_node_assert7.default)(!this._isStarted || this._isPaused, "Invalid state.");
+    this._state._startTimeframe = timeframe;
     this._timeframeClock.setTimeframe(timeframe);
+    for (const [key, seq] of timeframe.frames()) {
+      const feed = this._feeds.get(key);
+      if (!feed) {
+        throw new Error("Feed not found");
+      }
+      feed.undownload({
+        callback: () => (0, import_log7.log)("Undownloaded", {}, {
+          file: "pipeline.ts",
+          line: 297,
+          scope: this,
+          callSite: (f, a) => f(...a)
+        })
+      });
+      feed.download({
+        start: seq + 1,
+        linear: true
+      }).catch((err) => {
+        (0, import_log7.log)("failed to download feed", {
+          err
+        }, {
+          file: "pipeline.ts",
+          line: 300,
+          scope: this,
+          callSite: (f, a) => f(...a)
+        });
+      });
+    }
     if (this._feedSetIterator) {
       await this._feedSetIterator.close();
       await this._initIterator();
@@ -1047,7 +1089,7 @@ var Pipeline = class {
     this._feedSetIterator.stalled.on((iterator) => {
       import_log7.log.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, {}, {
         file: "pipeline.ts",
-        line: 357,
+        line: 381,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -1075,9 +1117,13 @@ __decorate4([
 ], Pipeline.prototype, "unpause", null);
 
 // packages/core/echo/echo-pipeline/src/space/control-pipeline.ts
+var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
 var ControlPipeline = class {
-  constructor({ spaceKey, genesisFeed, feedProvider }) {
+  constructor({ spaceKey, genesisFeed, feedProvider, metadataStore }) {
+    this._lastTimeframeSaveTime = Date.now();
     this.onFeedAdmitted = new import_util4.Callback();
+    this._spaceKey = spaceKey;
+    this._metadata = metadataStore;
     this._pipeline = new Pipeline();
     void this._pipeline.addFeed(genesisFeed);
     this._spaceStateMachine = new import_credentials.SpaceStateMachine(spaceKey);
@@ -1086,7 +1132,7 @@ var ControlPipeline = class {
         key: info.key
       }, {
         file: "control-pipeline.ts",
-        line: 39,
+        line: 51,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -1097,7 +1143,7 @@ var ControlPipeline = class {
         } catch (err) {
           import_log8.log.catch(err, {}, {
             file: "control-pipeline.ts",
-            line: 47,
+            line: 59,
             scope: this,
             callSite: (f, a) => f(...a)
           });
@@ -1120,7 +1166,7 @@ var ControlPipeline = class {
   async start() {
     (0, import_log8.log)("starting...", {}, {
       file: "control-pipeline.ts",
-      line: 71,
+      line: 83,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -1132,7 +1178,7 @@ var ControlPipeline = class {
             seq: msg.seq
           }, {
             file: "control-pipeline.ts",
-            line: 76,
+            line: 88,
             scope: this,
             callSite: (f, a) => f(...a)
           });
@@ -1143,16 +1189,18 @@ var ControlPipeline = class {
                 msg
               }, {
                 file: "control-pipeline.ts",
-                line: 83,
+                line: 95,
                 scope: this,
                 callSite: (f, a) => f(...a)
               });
+            } else {
+              await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
             }
           }
         } catch (err) {
           import_log8.log.catch(err, {}, {
             file: "control-pipeline.ts",
-            line: 87,
+            line: 101,
             scope: this,
             callSite: (f, a) => f(...a)
           });
@@ -1162,36 +1210,59 @@ var ControlPipeline = class {
     await this._pipeline.start();
     (0, import_log8.log)("started", {}, {
       file: "control-pipeline.ts",
-      line: 93,
+      line: 107,
       scope: this,
       callSite: (f, a) => f(...a)
     });
   }
+  async _noteTargetStateIfNeeded(timeframe) {
+    if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL) {
+      this._lastTimeframeSaveTime = Date.now();
+      await this._saveTargetTimeframe(timeframe);
+    }
+  }
   async stop() {
     (0, import_log8.log)("stopping...", {}, {
       file: "control-pipeline.ts",
-      line: 97,
+      line: 121,
       scope: this,
       callSite: (f, a) => f(...a)
     });
     await this._pipeline.stop();
+    await this._saveTargetTimeframe(this._pipeline.state.timeframe);
     (0, import_log8.log)("stopped", {}, {
       file: "control-pipeline.ts",
-      line: 99,
+      line: 124,
       scope: this,
       callSite: (f, a) => f(...a)
     });
   }
+  async _saveTargetTimeframe(timeframe) {
+    var _a;
+    try {
+      const newTimeframe = import_timeframe3.Timeframe.merge((_a = this._targetTimeframe) != null ? _a : new import_timeframe3.Timeframe(), timeframe);
+      await this._metadata.setSpaceControlLatestTimeframe(this._spaceKey, newTimeframe);
+      this._targetTimeframe = newTimeframe;
+    } catch (err) {
+      (0, import_log8.log)(err, {}, {
+        file: "control-pipeline.ts",
+        line: 133,
+        scope: this,
+        callSite: (f, a) => f(...a)
+      });
+    }
+  }
 };
 
 // packages/core/echo/echo-pipeline/src/space/data-pipeline.ts
 var import_node_assert8 = __toESM(require("node:assert"));
 var import_async6 = require("@dxos/async");
-var import_context3 = require("@dxos/context");
+var import_context4 = require("@dxos/context");
 var import_credentials3 = require("@dxos/credentials");
 var import_echo_db3 = require("@dxos/echo-db");
+var import_errors2 = require("@dxos/errors");
 var import_log9 = require("@dxos/log");
-var import_timeframe3 = require("@dxos/timeframe");
+var import_timeframe4 = require("@dxos/timeframe");
 var __decorate5 = function(decorators, target, key, desc) {
   var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
   if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
@@ -1204,15 +1275,16 @@ var __decorate5 = function(decorators, target, key, desc) {
 };
 var MESSAGES_PER_SNAPSHOT = 10;
 var AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL = 5e3;
-var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
+var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL2 = 500;
 var DataPipeline = class DataPipeline2 {
   constructor(_params) {
     this._params = _params;
-    this._ctx = new import_context3.Context();
-    this._lastAutomaticSnapshotTimeframe = new import_timeframe3.Timeframe();
+    this._ctx = new import_context4.Context();
+    this._lastAutomaticSnapshotTimeframe = new import_timeframe4.Timeframe();
     this._isOpen = false;
     this._lastTimeframeSaveTime = 0;
     this._lastSnapshotSaveTime = 0;
+    this._lastProcessedEpoch = -1;
     this.onNewEpoch = new import_async6.Event();
   }
   get isOpen() {
@@ -1236,17 +1308,10 @@ var DataPipeline = class DataPipeline2 {
         if (!(0, import_credentials3.checkCredentialType)(credential, "dxos.halo.credentials.Epoch")) {
           return;
         }
-        (0, import_log9.log)("new epoch", {
-          credential
-        }, {
-          file: "data-pipeline.ts",
-          line: 109,
-          scope: this,
-          callSite: (f, a) => f(...a)
-        });
-        await this._processEpoch(credential.subject.assertion);
         this.currentEpoch = credential;
-        this.onNewEpoch.emit(credential);
+        if (this._isOpen) {
+          await this._processEpochInSeparateTask(credential);
+        }
       }
     };
   }
@@ -1285,7 +1350,7 @@ var DataPipeline = class DataPipeline2 {
     }
     (0, import_log9.log)("close", {}, {
       file: "data-pipeline.ts",
-      line: 161,
+      line: 174,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -1300,7 +1365,7 @@ var DataPipeline = class DataPipeline2 {
     } catch (err) {
       import_log9.log.catch(err, {}, {
         file: "data-pipeline.ts",
-        line: 174,
+        line: 187,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -1310,6 +1375,11 @@ var DataPipeline = class DataPipeline2 {
     await this._params.snapshotManager.close();
   }
   async _consumePipeline() {
+    if (this.currentEpoch) {
+      const waitForOneEpoch = this.onNewEpoch.waitForCount(1);
+      await this._processEpochInSeparateTask(this.currentEpoch);
+      await waitForOneEpoch;
+    }
     (0, import_node_assert8.default)(this._pipeline, "Pipeline is not initialized.");
     for await (const msg of this._pipeline.consume()) {
       const { feedKey, seq, data } = msg;
@@ -1318,7 +1388,7 @@ var DataPipeline = class DataPipeline2 {
         seq
       }, {
         file: "data-pipeline.ts",
-        line: 186,
+        line: 205,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -1330,7 +1400,7 @@ var DataPipeline = class DataPipeline2 {
               feedKey
             }, {
               file: "data-pipeline.ts",
-              line: 192,
+              line: 211,
               scope: this,
               callSite: (f, a) => f(...a)
             });
@@ -1345,12 +1415,22 @@ var DataPipeline = class DataPipeline2 {
               memberKey: feedInfo.assertion.identityKey
             }
           });
+          import_log9.log.trace("dxos.echo.data-pipeline.processed", {
+            feedKey: feedKey.toHex(),
+            seq,
+            spaceKey: this._params.spaceKey.toHex()
+          }, {
+            file: "data-pipeline.ts",
+            line: 225,
+            scope: this,
+            callSite: (f, a) => f(...a)
+          });
           await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
         }
       } catch (err) {
         import_log9.log.catch(err, {}, {
           file: "data-pipeline.ts",
-          line: 210,
+          line: 235,
           scope: this,
           callSite: (f, a) => f(...a)
         });
@@ -1367,8 +1447,8 @@ var DataPipeline = class DataPipeline2 {
   }
   async _saveTargetTimeframe(timeframe) {
     var _a;
-    const newTimeframe = import_timeframe3.Timeframe.merge((_a = this._targetTimeframe) != null ? _a : new import_timeframe3.Timeframe(), timeframe);
-    await this._params.metadataStore.setSpaceLatestTimeframe(this._params.spaceKey, newTimeframe);
+    const newTimeframe = import_timeframe4.Timeframe.merge((_a = this._targetTimeframe) != null ? _a : new import_timeframe4.Timeframe(), timeframe);
+    await this._params.metadataStore.setSpaceDataLatestTimeframe(this._params.spaceKey, newTimeframe);
     this._targetTimeframe = newTimeframe;
   }
   async _saveCache() {
@@ -1385,7 +1465,7 @@ var DataPipeline = class DataPipeline2 {
     } catch (err) {
       import_log9.log.warn("Failed to cache properties", err, {
         file: "data-pipeline.ts",
-        line: 244,
+        line: 269,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -1393,7 +1473,7 @@ var DataPipeline = class DataPipeline2 {
     await this._params.metadataStore.setCache(this._params.spaceKey, cache);
   }
   async _noteTargetStateIfNeeded(timeframe) {
-    if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL) {
+    if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL2) {
       this._lastTimeframeSaveTime = Date.now();
       await this._saveTargetTimeframe(timeframe);
     }
@@ -1401,24 +1481,68 @@ var DataPipeline = class DataPipeline2 {
       await this._saveCache();
     }
   }
-  async _processEpoch(epoch) {
-    (0, import_node_assert8.default)(this._isOpen);
+  async _processEpochInSeparateTask(epoch) {
+    var _a;
+    if (epoch.subject.assertion.number <= this._lastProcessedEpoch) {
+      return;
+    }
+    await ((_a = this._epochCtx) == null ? void 0 : _a.dispose());
+    const ctx = new import_context4.Context({
+      onError: (err) => {
+        if (err instanceof import_errors2.CancelledError) {
+          (0, import_log9.log)("Epoch processing cancelled.", {}, {
+            file: "data-pipeline.ts",
+            line: 301,
+            scope: this,
+            callSite: (f, a) => f(...a)
+          });
+        } else {
+          import_log9.log.catch(err, {}, {
+            file: "data-pipeline.ts",
+            line: 303,
+            scope: this,
+            callSite: (f, a) => f(...a)
+          });
+        }
+      }
+    });
+    this._epochCtx = ctx;
+    (0, import_async6.scheduleTask)(ctx, async () => {
+      if (!this._isOpen) {
+        return;
+      }
+      (0, import_log9.log)("process epoch", {
+        epoch
+      }, {
+        file: "data-pipeline.ts",
+        line: 313,
+        scope: this,
+        callSite: (f, a) => f(...a)
+      });
+      await this._processEpoch(ctx, epoch.subject.assertion);
+      this.appliedEpoch = epoch;
+      this.onNewEpoch.emit(epoch);
+    });
+  }
+  async _processEpoch(ctx, epoch) {
+    (0, import_node_assert8.default)(this._isOpen, "Space is closed.");
     (0, import_node_assert8.default)(this._pipeline);
+    this._lastProcessedEpoch = epoch.number;
     (0, import_log9.log)("Processing epoch", {
       epoch
     }, {
       file: "data-pipeline.ts",
-      line: 273,
+      line: 327,
       scope: this,
       callSite: (f, a) => f(...a)
     });
     if (epoch.snapshotCid) {
-      const snapshot = await this._params.snapshotManager.load(epoch.snapshotCid);
+      const snapshot = await this._params.snapshotManager.load(ctx, epoch.snapshotCid);
       this.databaseHost._itemDemuxer.restoreFromSnapshot(snapshot.database);
     }
     (0, import_log9.log)("restarting pipeline for epoch", {}, {
       file: "data-pipeline.ts",
-      line: 281,
+      line: 335,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -1492,20 +1616,26 @@ var Space = class Space2 {
     this._controlPipeline = new ControlPipeline({
       spaceKey: params.spaceKey,
       genesisFeed: params.genesisFeed,
-      feedProvider: params.feedProvider
+      feedProvider: params.feedProvider,
+      metadataStore: params.metadataStore
     });
     this._controlPipeline.onFeedAdmitted.set(async (info) => {
+      const sparse = info.assertion.designation === import_credentials4.AdmittedFeed.Designation.DATA;
       if (info.assertion.designation === import_credentials4.AdmittedFeed.Designation.DATA) {
         await this._addFeedLock.executeSynchronized(async () => {
           if (this._dataPipeline.pipeline) {
             if (!this._dataPipeline.pipeline.hasFeed(info.key)) {
-              return this._dataPipeline.pipeline.addFeed(await this._feedProvider(info.key));
+              return this._dataPipeline.pipeline.addFeed(await this._feedProvider(info.key, {
+                sparse
+              }));
             }
           }
         });
       }
       if (!info.key.equals(params.genesisFeed.key)) {
-        this.protocol.addFeed(await params.feedProvider(info.key));
+        this.protocol.addFeed(await params.feedProvider(info.key, {
+          sparse
+        }));
       }
     });
     this._controlPipeline.onCredentialProcessed.set(async (credential) => {
@@ -1514,7 +1644,7 @@ var Space = class Space2 {
         credential
       }, {
         file: "space.ts",
-        line: 106,
+        line: 110,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -1537,7 +1667,9 @@ var Space = class Space2 {
         await this._addFeedLock.executeSynchronized(async () => {
           for (const feed of this._controlPipeline.spaceState.feeds.values()) {
             if (feed.assertion.designation === import_credentials4.AdmittedFeed.Designation.DATA && !pipeline.hasFeed(feed.key)) {
-              await pipeline.addFeed(await this._feedProvider(feed.key));
+              await pipeline.addFeed(await this._feedProvider(feed.key, {
+                sparse: true
+              }));
             }
           }
         });
@@ -1604,7 +1736,7 @@ var Space = class Space2 {
   async open() {
     (0, import_log10.log)("opening...", {}, {
       file: "space.ts",
-      line: 208,
+      line: 212,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -1614,10 +1746,11 @@ var Space = class Space2 {
     await this._controlPipeline.start();
     await this.protocol.start();
     this._dataPipelineCredentialConsumer = this._controlPipeline.spaceState.registerProcessor(this._dataPipeline.createCredentialProcessor());
+    await this._dataPipelineCredentialConsumer.open();
     this._isOpen = true;
     (0, import_log10.log)("opened", {}, {
       file: "space.ts",
-      line: 222,
+      line: 228,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -1628,7 +1761,7 @@ var Space = class Space2 {
       key: this._key
     }, {
       file: "space.ts",
-      line: 227,
+      line: 233,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -1643,7 +1776,7 @@ var Space = class Space2 {
     this._isOpen = false;
     (0, import_log10.log)("closed", {}, {
       file: "space.ts",
-      line: 241,
+      line: 247,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -1651,14 +1784,12 @@ var Space = class Space2 {
   async initializeDataPipeline() {
     (0, import_log10.log)("initializeDataPipeline", {}, {
       file: "space.ts",
-      line: 246,
+      line: 252,
       scope: this,
       callSite: (f, a) => f(...a)
     });
     (0, import_node_assert9.default)(this._isOpen, "Space must be open to initialize data pipeline.");
     await this._dataPipeline.open();
-    (0, import_node_assert9.default)(this._dataPipelineCredentialConsumer);
-    await this._dataPipelineCredentialConsumer.open();
   }
 };
 __decorate6([
@@ -1933,7 +2064,7 @@ var SpaceManager = class SpaceManager2 {
       spaceKey,
       protocol,
       genesisFeed,
-      feedProvider: (feedKey) => this._feedStore.openFeed(feedKey),
+      feedProvider: (feedKey, opts) => this._feedStore.openFeed(feedKey, opts),
       modelFactory: this._modelFactory,
       metadataStore: this._metadataStore,
       snapshotManager,
@@ -2063,7 +2194,8 @@ var TestAgent = class {
       writable: true
     });
     const dataFeed = await this.feedStore.openFeed(dataKey != null ? dataKey : await this.keyring.createKey(), {
-      writable: true
+      writable: true,
+      sparse: true
     });
     const metadata = {
       key: spaceKey,
@@ -2148,7 +2280,7 @@ var import_echo_db4 = require("@dxos/echo-db");
 var import_testing2 = require("@dxos/feed-store/testing");
 var import_keys7 = require("@dxos/keys");
 var import_model_factory2 = require("@dxos/model-factory");
-var import_timeframe4 = require("@dxos/timeframe");
+var import_timeframe5 = require("@dxos/timeframe");
 var createMemoryDatabase = async (modelFactory) => {
   const feed = new import_testing2.MockFeedWriter();
   const backend = new DatabaseHost(feed);
@@ -2157,7 +2289,7 @@ var createMemoryDatabase = async (modelFactory) => {
     meta: {
       ...meta,
       memberKey: import_keys7.PublicKey.random(),
-      timeframe: new import_timeframe4.Timeframe([
+      timeframe: new import_timeframe5.Timeframe([
         [
           meta.feedKey,
           meta.seq
@@ -2200,7 +2332,7 @@ var testLocalDatabase = async (create, check = create) => {
       ]
     }
   }));
-  await (0, import_async9.asyncTimeout)(check.databaseHost._itemDemuxer.mutation.waitForCondition(() => check.itemManager.entities.has(objectId)), 500);
+  await (0, import_async9.asyncTimeout)(check.databaseHost._itemDemuxer.mutation.waitForCondition(() => check.itemManager.entities.has(objectId)), 1e3);
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {