@dxos/echo-pipeline 0.1.41 → 0.1.45-next.45667d4

package/dist/lib/node/testing/index.cjs

@@ -42,14 +42,16 @@ __export(testing_exports, {
 module.exports = __toCommonJS(testing_exports);
 
 // packages/core/echo/echo-pipeline/src/testing/test-agent-builder.ts
+var import_credentials5 = require("@dxos/credentials");
 var import_document_model = require("@dxos/document-model");
 var import_keys6 = require("@dxos/keys");
 var import_messaging = require("@dxos/messaging");
 var import_model_factory = require("@dxos/model-factory");
 var import_network_manager2 = require("@dxos/network-manager");
+var import_credentials6 = require("@dxos/protocols/proto/dxos/halo/credentials");
 var import_random_access_storage = require("@dxos/random-access-storage");
 var import_teleport_extension_gossip = require("@dxos/teleport-extension-gossip");
-var import_util7 = require("@dxos/util");
+var import_util8 = require("@dxos/util");
 
 // packages/core/echo/echo-pipeline/src/dbhost/data-service-host.ts
 var import_node_assert = __toESM(require("node:assert"));
@@ -160,9 +162,8 @@ var DataServiceHost = class {
 // packages/core/echo/echo-pipeline/src/dbhost/database-host.ts
 var import_echo_db2 = require("@dxos/echo-db");
 var DatabaseHost = class {
-  constructor(_outboundStream, _snapshot) {
+  constructor(_outboundStream) {
     this._outboundStream = _outboundStream;
-    this._snapshot = _snapshot;
   }
   get isReadOnly() {
     return !!this._outboundStream;
@@ -175,9 +176,6 @@ var DatabaseHost = class {
     this._itemManager._debugLabel = "host";
     this._itemDemuxer = new import_echo_db2.ItemDemuxer(itemManager, modelFactory);
     this._echoProcessor = this._itemDemuxer.open();
-    if (this._snapshot) {
-      this._itemDemuxer.restoreFromSnapshot(this._snapshot);
-    }
   }
   async close() {
   }
@@ -287,6 +285,16 @@ var SnapshotStore = class {
       await file.close();
     }
   }
+  async listSnapshots() {
+    const entries = await this._directory.list();
+    return await Promise.all(entries.map(async (key) => {
+      const { size } = await this._directory.getOrCreateFile(key).stat();
+      return {
+        key,
+        size
+      };
+    }));
+  }
 };
 
 // packages/core/echo/echo-pipeline/src/dbhost/data-service.ts
@@ -463,13 +471,18 @@ var MetadataStore = class {
       await file.close();
     }
   }
+  _getSpace(spaceKey) {
+    const space = this.spaces.find((space2) => space2.key === spaceKey);
+    (0, import_node_assert3.default)(space, "Space not found");
+    return space;
+  }
   /**
   * Clears storage - doesn't work for now.
   */
   async clear() {
     (0, import_log3.log)("clearing all metadata", {}, {
       file: "metadata-store.ts",
-      line: 126,
+      line: 132,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -490,20 +503,15 @@ var MetadataStore = class {
     await this._save();
   }
   async setSpaceLatestTimeframe(spaceKey, timeframe) {
-    const space = this.spaces.find((space2) => space2.key === spaceKey);
-    (0, import_node_assert3.default)(space, "Space not found");
-    space.dataTimeframe = timeframe;
+    this._getSpace(spaceKey).dataTimeframe = timeframe;
     await this._save();
   }
-  async setSpaceSnapshot(spaceKey, snapshot) {
-    const space = this.spaces.find((space2) => space2.key === spaceKey);
-    (0, import_node_assert3.default)(space, "Space not found");
-    space.snapshot = snapshot;
+  async setCache(spaceKey, cache) {
+    this._getSpace(spaceKey).cache = cache;
     await this._save();
   }
   async setWritableFeedKeys(spaceKey, controlFeedKey, dataFeedKey) {
-    const space = this.spaces.find((space2) => space2.key === spaceKey);
-    (0, import_node_assert3.default)(space, "Space not found");
+    const space = this._getSpace(spaceKey);
     space.controlFeedKey = controlFeedKey;
     space.dataFeedKey = dataFeedKey;
     await this._save();
@@ -535,7 +543,6 @@ var AuthExtension = class extends import_teleport.RpcExtension {
         AuthService: import_protocols4.schema.getService("dxos.mesh.teleport.auth.AuthService")
       },
       timeout: 60 * 1e3
-      // Long timeout because auth can wait for sync in certain cases.
     });
     this._authParams = _authParams;
     this._ctx = new import_context.Context({
@@ -605,20 +612,29 @@ var AuthExtension = class extends import_teleport.RpcExtension {
 };
 
 // packages/core/echo/echo-pipeline/src/space/space.ts
-var import_node_assert8 = __toESM(require("node:assert"));
-var import_async6 = require("@dxos/async");
-var import_log8 = require("@dxos/log");
-var import_credentials3 = require("@dxos/protocols/proto/dxos/halo/credentials");
+var import_node_assert9 = __toESM(require("node:assert"));
+var import_async7 = require("@dxos/async");
+var import_log9 = require("@dxos/log");
+var import_credentials4 = require("@dxos/protocols/proto/dxos/halo/credentials");
+var import_util5 = require("@dxos/util");
+
+// packages/core/echo/echo-pipeline/src/space/control-pipeline.ts
+var import_credentials = require("@dxos/credentials");
+var import_keys3 = require("@dxos/keys");
+var import_log7 = require("@dxos/log");
+var import_credentials2 = require("@dxos/protocols/proto/dxos/halo/credentials");
 var import_util4 = require("@dxos/util");
 
 // packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts
 var import_node_assert7 = __toESM(require("node:assert"));
 var import_async5 = require("@dxos/async");
 var import_context2 = require("@dxos/context");
+var import_debug4 = require("@dxos/debug");
 var import_feed_store = require("@dxos/feed-store");
 var import_keys2 = require("@dxos/keys");
 var import_log6 = require("@dxos/log");
 var import_timeframe2 = require("@dxos/timeframe");
+var import_util3 = require("@dxos/util");
 
 // packages/core/echo/echo-pipeline/src/common/codec.ts
 var import_hypercore = require("@dxos/hypercore");
@@ -696,6 +712,11 @@ var TimeframeClock = class {
   get pendingTimeframe() {
     return this._pendingTimeframe;
   }
+  setTimeframe(timeframe) {
+    this._timeframe = timeframe;
+    this._pendingTimeframe = timeframe;
+    this.update.emit(this._timeframe);
+  }
   updatePendingTimeframe(key, seq) {
     this._pendingTimeframe = import_timeframe.Timeframe.merge(this._pendingTimeframe, new import_timeframe.Timeframe([
       [
@@ -713,12 +734,12 @@ var TimeframeClock = class {
     return !gaps.isEmpty();
   }
   async waitUntilReached(target) {
-    import_log5.log.info("waitUntilReached", {
+    (0, import_log5.log)("waitUntilReached", {
       target,
       current: this._timeframe
     }, {
       file: "timeframe-clock.ts",
-      line: 67,
+      line: 73,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -729,7 +750,7 @@ var TimeframeClock = class {
         deps: import_timeframe.Timeframe.dependencies(target, this._timeframe)
       }, {
         file: "timeframe-clock.ts",
-        line: 69,
+        line: 75,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -754,10 +775,11 @@ var __decorate4 = function(decorators, target, key, desc) {
 };
 var PipelineState = class {
   // prettier-ignore
-  constructor(_iterator, _timeframeClock) {
-    this._iterator = _iterator;
+  constructor(_feeds, _timeframeClock) {
+    this._feeds = _feeds;
     this._timeframeClock = _timeframeClock;
     this.timeframeUpdate = this._timeframeClock.update;
+    this.stalled = new import_async5.Event();
   }
   /**
   * Latest theoretical timeframe based on the last mutation in each feed.
@@ -765,7 +787,7 @@ var PipelineState = class {
   */
   // TODO(dmaretskyi): Rename `totalTimeframe`? or `lastTimeframe`.
   get endTimeframe() {
-    return mapFeedIndexesToTimeframe(this._iterator.feeds.filter((feed) => feed.length > 0).map((feed) => ({
+    return mapFeedIndexesToTimeframe(Array.from(this._feeds.values()).filter((feed) => feed.length > 0).map((feed) => ({
       feedKey: feed.key,
       index: feed.length - 1
     })));
@@ -780,7 +802,7 @@ var PipelineState = class {
     return this._targetTimeframe ? this._targetTimeframe : new import_timeframe2.Timeframe();
   }
   get feeds() {
-    return this._iterator.feeds;
+    return Array.from(this._feeds.values());
   }
   async waitUntilTimeframe(target) {
     await this._timeframeClock.waitUntilReached(target);
@@ -803,7 +825,7 @@ var PipelineState = class {
       target: this.targetTimeframe
     }, {
       file: "pipeline.ts",
-      line: 105,
+      line: 109,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -812,7 +834,7 @@ var PipelineState = class {
         return import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe).isEmpty();
       }),
       ...breakOnStall ? [
-        this._iterator.stalled.discardParameter().waitForCount(1)
+        this.stalled.discardParameter().waitForCount(1)
       ] : []
     ]);
     let done = false;
@@ -833,7 +855,7 @@ var PipelineState = class {
             dependencies: import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe)
           }, {
             file: "pipeline.ts",
-            line: 131,
+            line: 135,
             scope: this,
             callSite: (f, a) => f(...a)
           });
@@ -845,25 +867,18 @@ var PipelineState = class {
   }
 };
 var Pipeline = class {
-  // prettier-ignore
-  constructor(_initialTimeframe = new import_timeframe2.Timeframe()) {
-    this._initialTimeframe = _initialTimeframe;
-    this._timeframeClock = new TimeframeClock(this._initialTimeframe);
-    this._feedSetIterator = new import_feed_store.FeedSetIterator(createMessageSelector(this._timeframeClock), {
-      start: startAfter(this._initialTimeframe),
-      stallTimeout: 1e3
-    });
-    this._state = new PipelineState(this._feedSetIterator, this._timeframeClock);
+  constructor() {
+    this._timeframeClock = new TimeframeClock(new import_timeframe2.Timeframe());
+    this._feeds = new import_util3.ComplexMap(import_keys2.PublicKey.hash);
+    // External state accessor.
+    this._state = new PipelineState(this._feeds, this._timeframeClock);
+    // Waits for the message consumer to process the message and yield control back to the pipeline.
     this._processingTrigger = new import_async5.Trigger().wake();
+    this._pauseTrigger = new import_async5.Trigger().wake();
+    this._isStopping = false;
+    this._isStarted = false;
     this._isOpen = false;
-    this._feedSetIterator.stalled.on((iterator) => {
-      import_log6.log.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, {}, {
-        file: "pipeline.ts",
-        line: 207,
-        scope: this,
-        callSite: (f, a) => f(...a)
-      });
-    });
+    this._isPaused = false;
   }
   get state() {
     return this._state;
@@ -878,10 +893,13 @@ var Pipeline = class {
   // NOTE: This cannot be synchronized with `stop` because stop waits for the mutation processing to complete,
   // which might be opening feeds during the mutation processing, which w
   async addFeed(feed) {
-    await this._feedSetIterator.addFeed(feed);
+    this._feeds.set(feed.key, feed);
+    if (this._feedSetIterator) {
+      await this._feedSetIterator.addFeed(feed);
+    }
   }
   hasFeed(feedKey) {
-    return this._feedSetIterator.hasFeed(feedKey);
+    return this._feeds.has(feedKey);
   }
   setWriteFeed(feed) {
     (0, import_node_assert7.default)(!this._writer, "Writer already set.");
@@ -894,33 +912,68 @@ var Pipeline = class {
   async start() {
     (0, import_log6.log)("starting...", {}, {
       file: "pipeline.ts",
-      line: 249,
+      line: 254,
       scope: this,
       callSite: (f, a) => f(...a)
     });
+    await this._initIterator();
     await this._feedSetIterator.open();
     (0, import_log6.log)("started", {}, {
       file: "pipeline.ts",
-      line: 251,
+      line: 257,
       scope: this,
       callSite: (f, a) => f(...a)
     });
+    this._isStarted = true;
   }
   async stop() {
+    var _a;
     (0, import_log6.log)("stopping...", {}, {
       file: "pipeline.ts",
-      line: 256,
+      line: 263,
       scope: this,
       callSite: (f, a) => f(...a)
     });
-    await this._feedSetIterator.close();
+    this._isStopping = true;
+    await ((_a = this._feedSetIterator) == null ? void 0 : _a.close());
     await this._processingTrigger.wait();
     (0, import_log6.log)("stopped", {}, {
       file: "pipeline.ts",
-      line: 259,
+      line: 267,
       scope: this,
       callSite: (f, a) => f(...a)
     });
+    this._isStarted = false;
+  }
+  /**
+  * @param timeframe Timeframe of already processed messages. The pipeline will start processing messages AFTER this timeframe.
+  */
+  async setCursor(timeframe) {
+    (0, import_node_assert7.default)(!this._isStarted || this._isPaused, "Invalid state.");
+    this._timeframeClock.setTimeframe(timeframe);
+    if (this._feedSetIterator) {
+      await this._feedSetIterator.close();
+      await this._initIterator();
+      await this._feedSetIterator.open();
+    }
+  }
+  /**
+  * Calling pause while processing will cause a deadlock.
+  */
+  async pause() {
+    (0, import_node_assert7.default)(this._isStarted, "Pipeline is not open.");
+    if (this._isPaused) {
+      return;
+    }
+    this._pauseTrigger.reset();
+    await this._processingTrigger.wait();
+    this._isPaused = true;
+  }
+  async unpause() {
+    (0, import_node_assert7.default)(this._isStarted, "Pipeline is not open.");
+    (0, import_node_assert7.default)(this._isPaused, "Pipeline is not paused.");
+    this._pauseTrigger.wake();
+    this._isPaused = false;
   }
   /**
   * Starts to iterate over the ordered messages from the added feeds.
@@ -929,7 +982,21 @@ var Pipeline = class {
   async *consume() {
     (0, import_node_assert7.default)(!this._isOpen, "Pipeline is already being consumed.");
     this._isOpen = true;
-    for await (const block of this._feedSetIterator) {
+    (0, import_node_assert7.default)(this._feedSetIterator, "Iterator not initialized.");
+    let lastFeedSetIterator = this._feedSetIterator;
+    let iterable = lastFeedSetIterator[Symbol.asyncIterator]();
+    while (!this._isStopping) {
+      await this._pauseTrigger.wait();
+      if (lastFeedSetIterator !== this._feedSetIterator) {
+        (0, import_node_assert7.default)(this._feedSetIterator, "Iterator not initialized.");
+        lastFeedSetIterator = this._feedSetIterator;
+        iterable = lastFeedSetIterator[Symbol.asyncIterator]();
+      }
+      const { done, value } = await iterable.next();
+      if (done) {
+        continue;
+      }
+      const block = value != null ? value : (0, import_debug4.failUndefined)();
       this._processingTrigger.reset();
       this._timeframeClock.updatePendingTimeframe(import_keys2.PublicKey.from(block.feedKey), block.seq);
       yield block;
@@ -938,6 +1005,24 @@ var Pipeline = class {
     }
     this._isOpen = false;
   }
+  async _initIterator() {
+    this._feedSetIterator = new import_feed_store.FeedSetIterator(createMessageSelector(this._timeframeClock), {
+      start: startAfter(this._timeframeClock.timeframe),
+      stallTimeout: 1e3
+    });
+    this._feedSetIterator.stalled.on((iterator) => {
+      import_log6.log.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, {}, {
+        file: "pipeline.ts",
+        line: 357,
+        scope: this,
+        callSite: (f, a) => f(...a)
+      });
+      this._state.stalled.emit();
+    });
+    for (const feed of this._feeds.values()) {
+      await this._feedSetIterator.addFeed(feed);
+    }
+  }
 };
 __decorate4([
   import_async5.synchronized
@@ -945,17 +1030,21 @@ __decorate4([
 __decorate4([
   import_async5.synchronized
 ], Pipeline.prototype, "stop", null);
+__decorate4([
+  import_async5.synchronized
+], Pipeline.prototype, "setCursor", null);
+__decorate4([
+  import_async5.synchronized
+], Pipeline.prototype, "pause", null);
+__decorate4([
+  import_async5.synchronized
+], Pipeline.prototype, "unpause", null);
 
 // packages/core/echo/echo-pipeline/src/space/control-pipeline.ts
-var import_credentials = require("@dxos/credentials");
-var import_keys3 = require("@dxos/keys");
-var import_log7 = require("@dxos/log");
-var import_credentials2 = require("@dxos/protocols/proto/dxos/halo/credentials");
-var import_util3 = require("@dxos/util");
 var ControlPipeline = class {
-  constructor({ spaceKey, genesisFeed, feedProvider, initialTimeframe }) {
-    this.onFeedAdmitted = new import_util3.Callback();
-    this._pipeline = new Pipeline(initialTimeframe);
+  constructor({ spaceKey, genesisFeed, feedProvider }) {
+    this.onFeedAdmitted = new import_util4.Callback();
+    this._pipeline = new Pipeline();
     void this._pipeline.addFeed(genesisFeed);
     this._spaceStateMachine = new import_credentials.SpaceStateMachine(spaceKey);
     this._spaceStateMachine.onFeedAdmitted.set(async (info) => {
@@ -963,7 +1052,7 @@ var ControlPipeline = class {
         key: info.key
       }, {
         file: "control-pipeline.ts",
-        line: 41,
+        line: 39,
         scope: this,
         callSite: (f, a) => f(...a)
       });
@@ -974,7 +1063,7 @@ var ControlPipeline = class {
         } catch (err) {
           import_log7.log.catch(err, {}, {
             file: "control-pipeline.ts",
-            line: 49,
+            line: 47,
             scope: this,
             callSite: (f, a) => f(...a)
           });
@@ -997,7 +1086,7 @@ var ControlPipeline = class {
   async start() {
     (0, import_log7.log)("starting...", {}, {
       file: "control-pipeline.ts",
-      line: 73,
+      line: 71,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -1009,7 +1098,7 @@ var ControlPipeline = class {
             seq: msg.seq
           }, {
             file: "control-pipeline.ts",
-            line: 78,
+            line: 76,
             scope: this,
             callSite: (f, a) => f(...a)
           });
@@ -1020,7 +1109,7 @@ var ControlPipeline = class {
                 msg
               }, {
                 file: "control-pipeline.ts",
-                line: 85,
+                line: 83,
                 scope: this,
                 callSite: (f, a) => f(...a)
               });
@@ -1029,7 +1118,7 @@ var ControlPipeline = class {
         } catch (err) {
           import_log7.log.catch(err, {}, {
             file: "control-pipeline.ts",
-            line: 89,
+            line: 87,
             scope: this,
             callSite: (f, a) => f(...a)
           });
@@ -1039,7 +1128,7 @@ var ControlPipeline = class {
     await this._pipeline.start();
     (0, import_log7.log)("started", {}, {
       file: "control-pipeline.ts",
-      line: 95,
+      line: 93,
       scope: this,
       callSite: (f, a) => f(...a)
     });
@@ -1047,21 +1136,28 @@ var ControlPipeline = class {
   async stop() {
     (0, import_log7.log)("stopping...", {}, {
       file: "control-pipeline.ts",
-      line: 99,
+      line: 97,
       scope: this,
       callSite: (f, a) => f(...a)
     });
     await this._pipeline.stop();
     (0, import_log7.log)("stopped", {}, {
       file: "control-pipeline.ts",
-      line: 101,
+      line: 99,
       scope: this,
       callSite: (f, a) => f(...a)
     });
   }
 };
 
-// packages/core/echo/echo-pipeline/src/space/space.ts
+// packages/core/echo/echo-pipeline/src/space/data-pipeline.ts
+var import_node_assert8 = __toESM(require("node:assert"));
+var import_async6 = require("@dxos/async");
+var import_context3 = require("@dxos/context");
+var import_credentials3 = require("@dxos/credentials");
+var import_echo_db3 = require("@dxos/echo-db");
+var import_log8 = require("@dxos/log");
+var import_timeframe3 = require("@dxos/timeframe");
 var __decorate5 = function(decorators, target, key, desc) {
   var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
   if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
@@ -1072,191 +1168,257 @@ var __decorate5 = function(decorators, target, key, desc) {
         r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
   return c > 3 && r && Object.defineProperty(target, key, r), r;
 };
-var Space = class Space2 {
-  constructor({ spaceKey, protocol, genesisFeed, feedProvider }) {
-    this._addFeedLock = new import_async6.Lock();
-    this.onCredentialProcessed = new import_util4.Callback();
-    this.stateUpdate = new import_async6.Event();
+var MESSAGES_PER_SNAPSHOT = 10;
+var AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL = 5e3;
+var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
+var DataPipeline = class DataPipeline2 {
+  constructor(_params) {
+    this._params = _params;
+    this._ctx = new import_context3.Context();
+    this._lastAutomaticSnapshotTimeframe = new import_timeframe3.Timeframe();
     this._isOpen = false;
-    (0, import_node_assert8.default)(spaceKey && feedProvider);
-    this._key = spaceKey;
-    this._genesisFeedKey = genesisFeed.key;
-    this._feedProvider = feedProvider;
-    this._controlPipeline = new ControlPipeline({
-      spaceKey,
-      genesisFeed,
-      feedProvider
-    });
-    this._controlPipeline.onFeedAdmitted.set(async (info) => {
-      if (info.assertion.designation === import_credentials3.AdmittedFeed.Designation.DATA) {
-        await this._addFeedLock.executeSynchronized(async () => {
-          if (this._dataPipeline) {
-            if (!this._dataPipeline.hasFeed(info.key)) {
-              return this._dataPipeline.addFeed(await this._feedProvider(info.key));
-            }
-          }
-        });
-      }
-      if (!info.key.equals(genesisFeed.key)) {
-        this.protocol.addFeed(await feedProvider(info.key));
-      }
-    });
-    this._controlPipeline.onCredentialProcessed.set(async (credential) => {
-      await this.onCredentialProcessed.callIfSet(credential);
-      (0, import_log8.log)("onCredentialProcessed", {
-        credential
-      }, {
-        file: "space.ts",
-        line: 87,
-        scope: this,
-        callSite: (f, a) => f(...a)
-      });
-      this.stateUpdate.emit();
-    });
-    this.protocol = protocol;
-    this.protocol.addFeed(genesisFeed);
-  }
-  get key() {
-    return this._key;
+    this._lastTimeframeSaveTime = 0;
+    this._lastSnapshotSaveTime = 0;
+    this.onNewEpoch = new import_async6.Event();
   }
   get isOpen() {
     return this._isOpen;
   }
-  get genesisFeedKey() {
-    return this._genesisFeedKey;
-  }
-  get controlFeedKey() {
-    var _a;
-    return (_a = this._controlFeed) == null ? void 0 : _a.key;
+  get pipeline() {
+    return this._pipeline;
   }
-  get dataFeedKey() {
+  get pipelineState() {
     var _a;
-    return (_a = this._dataFeed) == null ? void 0 : _a.key;
-  }
-  get spaceState() {
-    return this._controlPipeline.spaceState;
-  }
-  /**
-  * @test-only
-  */
-  get controlPipeline() {
-    return this._controlPipeline.pipeline;
-  }
-  setControlFeed(feed) {
-    (0, import_node_assert8.default)(!this._controlFeed, "Control feed already set.");
-    this._controlFeed = feed;
-    this._controlPipeline.setWriteFeed(feed);
-    return this;
+    return (_a = this._pipeline) == null ? void 0 : _a.state;
   }
-  setDataFeed(feed) {
+  setTargetTimeframe(timeframe) {
     var _a;
-    (0, import_node_assert8.default)(!this._dataFeed, "Data feed already set.");
-    this._dataFeed = feed;
-    (_a = this._dataPipeline) == null ? void 0 : _a.setWriteFeed(feed);
-    return this;
+    this._targetTimeframe = timeframe;
+    (_a = this._pipeline) == null ? void 0 : _a.state.setTargetTimeframe(timeframe);
   }
-  /**
-  * Use for diagnostics.
-  */
-  getControlFeeds() {
-    return Array.from(this._controlPipeline.spaceState.feeds.values());
+  createCredentialProcessor() {
+    return {
+      process: async (credential) => {
+        const assertion = (0, import_credentials3.getCredentialAssertion)(credential);
+        if (assertion["@type"] !== "dxos.halo.credentials.Epoch") {
+          return;
+        }
+        (0, import_log8.log)("new epoch", {
+          credential
+        }, {
+          file: "data-pipeline.ts",
+          line: 110,
+          scope: this,
+          callSite: (f, a) => f(...a)
+        });
+        await this._processEpoch(assertion);
+        this.currentEpoch = credential;
+        this.onNewEpoch.emit(credential);
+      }
+    };
   }
-  /**
-  * Use for diagnostics.
-  */
-  // getDataFeeds(): FeedInfo[] {
-  //   return this._dataPipeline?.getFeeds();
-  // }
   async open() {
-    (0, import_log8.log)("opening...", {}, {
-      file: "space.ts",
-      line: 157,
-      scope: this,
-      callSite: (f, a) => f(...a)
-    });
     if (this._isOpen) {
       return;
     }
-    await this._controlPipeline.start();
-    await this.protocol.start();
-    this._isOpen = true;
-    (0, import_log8.log)("opened", {}, {
-      file: "space.ts",
-      line: 167,
-      scope: this,
-      callSite: (f, a) => f(...a)
+    this._pipeline = new Pipeline();
+    await this._params.onPipelineCreated(this._pipeline);
+    await this._pipeline.start();
+    await this._pipeline.pause();
+    if (this._targetTimeframe) {
+      this._pipeline.state.setTargetTimeframe(this._targetTimeframe);
+    }
+    const feedWriter = {
+      write: (data, options) => {
+        (0, import_node_assert8.default)(this._pipeline, "Pipeline is not initialized.");
+        (0, import_node_assert8.default)(this.currentEpoch, "Epoch is not initialized.");
+        return this._pipeline.writer.write({
+          data
+        }, options);
+      }
+    };
+    this.databaseHost = new DatabaseHost(feedWriter);
+    this.itemManager = new import_echo_db3.ItemManager(this._params.modelFactory);
+    await this.databaseHost.open(this.itemManager, this._params.modelFactory);
+    (0, import_async6.scheduleTask)(this._ctx, async () => {
+      await this._consumePipeline();
     });
+    this._isOpen = true;
   }
   async close() {
-    (0, import_log8.log)("closing...", {
-      key: this._key
-    }, {
-      file: "space.ts",
-      line: 172,
-      scope: this,
-      callSite: (f, a) => f(...a)
-    });
+    var _a, _b, _c;
     if (!this._isOpen) {
       return;
     }
-    await this.protocol.stop();
-    await this._controlPipeline.stop();
-    this._isOpen = false;
-    (0, import_log8.log)("closed", {}, {
-      file: "space.ts",
-      line: 182,
+    (0, import_log8.log)("close", {}, {
+      file: "data-pipeline.ts",
+      line: 162,
       scope: this,
       callSite: (f, a) => f(...a)
     });
-  }
-  // TODO(dmaretskyi): Make reusable.
-  async createDataPipeline({ start }) {
-    (0, import_node_assert8.default)(this._isOpen, "Space must be open to initialize data pipeline.");
-    (0, import_node_assert8.default)(!this._dataPipeline, "Data pipeline already initialized.");
-    const pipeline = new Pipeline(start);
-    if (this._dataFeed) {
-      pipeline.setWriteFeed(this._dataFeed);
-    }
-    this._dataPipeline = pipeline;
-    await this._addFeedLock.executeSynchronized(async () => {
-      for (const feed of this._controlPipeline.spaceState.feeds.values()) {
-        if (feed.assertion.designation === import_credentials3.AdmittedFeed.Designation.DATA && !pipeline.hasFeed(feed.key)) {
-          await pipeline.addFeed(await this._feedProvider(feed.key));
-        }
+    this._isOpen = false;
+    await this._ctx.dispose();
+    await ((_a = this._pipeline) == null ? void 0 : _a.stop());
+    try {
+      await this._saveCache();
+      if (this._pipeline) {
+        await this._saveTargetTimeframe(this._pipeline.state.timeframe);
       }
-    });
-    return pipeline;
+    } catch (err) {
+      import_log8.log.catch(err, {}, {
+        file: "data-pipeline.ts",
+        line: 175,
+        scope: this,
+        callSite: (f, a) => f(...a)
+      });
+    }
+    await ((_b = this.databaseHost) == null ? void 0 : _b.close());
+    await ((_c = this.itemManager) == null ? void 0 : _c.destroy());
+    await this._params.snapshotManager.close();
   }
-};
-__decorate5([
-  import_log8.logInfo
-], Space.prototype, "key", null);
-__decorate5([
-  import_async6.synchronized
-], Space.prototype, "open", null);
-__decorate5([
-  import_async6.synchronized
-], Space.prototype, "close", null);
-Space = __decorate5([
-  (0, import_async6.trackLeaks)("open", "close")
-], Space);
-
-// packages/core/echo/echo-pipeline/src/space/space-manager.ts
-var import_async7 = require("@dxos/async");
-var import_debug4 = require("@dxos/debug");
-var import_keys5 = require("@dxos/keys");
-var import_log10 = require("@dxos/log");
-var import_protocols6 = require("@dxos/protocols");
-var import_util6 = require("@dxos/util");
-
-// packages/core/echo/echo-pipeline/src/space/space-protocol.ts
-var import_crypto3 = require("@dxos/crypto");
-var import_keys4 = require("@dxos/keys");
-var import_log9 = require("@dxos/log");
-var import_network_manager = require("@dxos/network-manager");
-var import_teleport2 = require("@dxos/teleport");
-var import_teleport_extension_replicator = require("@dxos/teleport-extension-replicator");
-var import_util5 = require("@dxos/util");
+  async _consumePipeline() {
+    (0, import_node_assert8.default)(this._pipeline, "Pipeline is not initialized.");
+    for await (const msg of this._pipeline.consume()) {
+      const { feedKey, seq, data } = msg;
+      (0, import_log8.log)("processing message", {
+        feedKey,
+        seq
+      }, {
+        file: "data-pipeline.ts",
+        line: 187,
+        scope: this,
+        callSite: (f, a) => f(...a)
+      });
+      try {
+        if (data.payload.data) {
+          const feedInfo = this._params.feedInfoProvider(feedKey);
+          if (!feedInfo) {
+            import_log8.log.error("Could not find feed.", {
+              feedKey
+            }, {
+              file: "data-pipeline.ts",
+              line: 193,
+              scope: this,
+              callSite: (f, a) => f(...a)
+            });
+            continue;
+          }
+          await this.databaseHost.echoProcessor({
+            batch: data.payload.data.batch,
+            meta: {
+              feedKey,
+              seq,
+              timeframe: data.timeframe,
+              memberKey: feedInfo.assertion.identityKey
+            }
+          });
+          await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
+        }
+      } catch (err) {
+        import_log8.log.catch(err, {}, {
+          file: "data-pipeline.ts",
+          line: 211,
+          scope: this,
+          callSite: (f, a) => f(...a)
+        });
+      }
+    }
+  }
+  _createSnapshot() {
+    (0, import_node_assert8.default)(this.databaseHost, "Database backend is not initialized.");
+    return {
+      spaceKey: this._params.spaceKey.asUint8Array(),
+      timeframe: this._pipeline.state.timeframe,
+      database: this.databaseHost.createSnapshot()
+    };
+  }
+  async _saveTargetTimeframe(timeframe) {
+    var _a;
+    const newTimeframe = import_timeframe3.Timeframe.merge((_a = this._targetTimeframe) != null ? _a : new import_timeframe3.Timeframe(), timeframe);
+    await this._params.metadataStore.setSpaceLatestTimeframe(this._params.spaceKey, newTimeframe);
+    this._targetTimeframe = newTimeframe;
+  }
+  async _saveCache() {
+    var _a;
+    const cache = {};
+    try {
+      const propertiesItem = this.itemManager.items.find((item) => {
+        var _a2, _b;
+        return ((_a2 = item.modelMeta) == null ? void 0 : _a2.type) === "dxos:model/document" && ((_b = (0, import_echo_db3.getStateMachineFromItem)(item)) == null ? void 0 : _b.snapshot().type) === "dxos.sdk.client.Properties";
+      });
+      if (propertiesItem) {
+        cache.properties = (_a = (0, import_echo_db3.getStateMachineFromItem)(propertiesItem)) == null ? void 0 : _a.snapshot();
+      }
+    } catch (err) {
+      import_log8.log.warn("Failed to cache properties", err, {
+        file: "data-pipeline.ts",
+        line: 245,
+        scope: this,
+        callSite: (f, a) => f(...a)
+      });
+    }
+    await this._params.metadataStore.setCache(this._params.spaceKey, cache);
+  }
+  async _noteTargetStateIfNeeded(timeframe) {
+    if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL) {
+      this._lastTimeframeSaveTime = Date.now();
+      await this._saveTargetTimeframe(timeframe);
+    }
+    if (Date.now() - this._lastSnapshotSaveTime > AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL && timeframe.totalMessages() - this._lastAutomaticSnapshotTimeframe.totalMessages() > MESSAGES_PER_SNAPSHOT) {
+      await this._saveCache();
+    }
+  }
+  async _processEpoch(epoch) {
+    (0, import_node_assert8.default)(this._isOpen);
+    (0, import_node_assert8.default)(this._pipeline);
+    if (epoch.snapshotCid) {
+      const snapshot = await this._params.snapshotManager.load(epoch.snapshotCid);
+      this.databaseHost._itemDemuxer.restoreFromSnapshot(snapshot.database);
+    }
+    await this._pipeline.pause();
+    await this._pipeline.setCursor(epoch.timeframe);
+    await this._pipeline.unpause();
+  }
+  async waitUntilTimeframe(timeframe) {
+    (0, import_node_assert8.default)(this._pipeline, "Pipeline is not initialized.");
+    await this._pipeline.state.waitUntilTimeframe(timeframe);
+  }
+  async createEpoch() {
+    (0, import_node_assert8.default)(this._pipeline);
+    (0, import_node_assert8.default)(this.currentEpoch);
+    await this._pipeline.pause();
+    const snapshot = await this._createSnapshot();
+    const snapshotCid = await this._params.snapshotManager.store(snapshot);
+    const epoch = {
+      previousId: this.currentEpoch.id,
+      timeframe: this._pipeline.state.timeframe,
+      number: this.currentEpoch.subject.assertion.number + 1,
+      snapshotCid
+    };
+    await this._pipeline.unpause();
+    return epoch;
+  }
+  async ensureEpochInitialized() {
+    await this.onNewEpoch.waitForCondition(() => !!this.currentEpoch);
+  }
+};
+__decorate5([
+  import_async6.synchronized
+], DataPipeline.prototype, "open", null);
+__decorate5([
+  import_async6.synchronized
+], DataPipeline.prototype, "close", null);
+__decorate5([
+  import_async6.synchronized
+], DataPipeline.prototype, "_processEpoch", null);
+__decorate5([
+  import_async6.synchronized
+], DataPipeline.prototype, "createEpoch", null);
+DataPipeline = __decorate5([
+  (0, import_async6.trackLeaks)("open", "close")
+], DataPipeline);
+
+// packages/core/echo/echo-pipeline/src/space/space.ts
 var __decorate6 = function(decorators, target, key, desc) {
   var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
   if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
@@ -1267,12 +1429,235 @@ var __decorate6 = function(decorators, target, key, desc) {
         r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
   return c > 3 && r && Object.defineProperty(target, key, r), r;
 };
+var Space = class Space2 {
+  constructor(params) {
+    this._addFeedLock = new import_async7.Lock();
+    this.onCredentialProcessed = new import_util5.Callback();
+    this.stateUpdate = new import_async7.Event();
+    // Processes epoch credentials.
+    this._dataPipelineCredentialConsumer = void 0;
+    this._isOpen = false;
+    (0, import_node_assert9.default)(params.spaceKey && params.feedProvider);
+    this._key = params.spaceKey;
+    this._genesisFeedKey = params.genesisFeed.key;
+    this._feedProvider = params.feedProvider;
+    this._snapshotManager = params.snapshotManager;
+    this._controlPipeline = new ControlPipeline({
+      spaceKey: params.spaceKey,
+      genesisFeed: params.genesisFeed,
+      feedProvider: params.feedProvider
+    });
+    this._controlPipeline.onFeedAdmitted.set(async (info) => {
+      if (info.assertion.designation === import_credentials4.AdmittedFeed.Designation.DATA) {
+        await this._addFeedLock.executeSynchronized(async () => {
+          if (this._dataPipeline.pipeline) {
+            if (!this._dataPipeline.pipeline.hasFeed(info.key)) {
+              return this._dataPipeline.pipeline.addFeed(await this._feedProvider(info.key));
+            }
+          }
+        });
+      }
+      if (!info.key.equals(params.genesisFeed.key)) {
+        this.protocol.addFeed(await params.feedProvider(info.key));
+      }
+    });
+    this._controlPipeline.onCredentialProcessed.set(async (credential) => {
+      await this.onCredentialProcessed.callIfSet(credential);
+      (0, import_log9.log)("onCredentialProcessed", {
+        credential
+      }, {
+        file: "space.ts",
+        line: 106,
+        scope: this,
+        callSite: (f, a) => f(...a)
+      });
+      this.stateUpdate.emit();
+    });
+    this.protocol = params.protocol;
+    this.protocol.addFeed(params.genesisFeed);
+    this._dataPipeline = new DataPipeline({
+      modelFactory: params.modelFactory,
+      metadataStore: params.metadataStore,
+      snapshotManager: params.snapshotManager,
+      memberKey: params.memberKey,
+      spaceKey: this._key,
+      feedInfoProvider: (feedKey) => this._controlPipeline.spaceState.feeds.get(feedKey),
+      snapshotId: params.snapshotId,
+      onPipelineCreated: async (pipeline) => {
+        if (this._dataFeed) {
+          pipeline.setWriteFeed(this._dataFeed);
+        }
+        await this._addFeedLock.executeSynchronized(async () => {
+          for (const feed of this._controlPipeline.spaceState.feeds.values()) {
+            if (feed.assertion.designation === import_credentials4.AdmittedFeed.Designation.DATA && !pipeline.hasFeed(feed.key)) {
+              await pipeline.addFeed(await this._feedProvider(feed.key));
+            }
+          }
+        });
+      }
+    });
+  }
+  get key() {
+    return this._key;
+  }
+  get isOpen() {
+    return this._isOpen;
+  }
+  get genesisFeedKey() {
+    return this._genesisFeedKey;
+  }
+  get controlFeedKey() {
+    var _a;
+    return (_a = this._controlFeed) == null ? void 0 : _a.key;
+  }
+  get dataFeedKey() {
+    var _a;
+    return (_a = this._dataFeed) == null ? void 0 : _a.key;
+  }
+  get spaceState() {
+    return this._controlPipeline.spaceState;
+  }
+  /**
+  * @test-only
+  */
+  get controlPipeline() {
+    return this._controlPipeline.pipeline;
+  }
+  get dataPipeline() {
+    return this._dataPipeline;
+  }
+  get snapshotManager() {
+    return this._snapshotManager;
+  }
+  setControlFeed(feed) {
+    (0, import_node_assert9.default)(!this._controlFeed, "Control feed already set.");
+    this._controlFeed = feed;
+    this._controlPipeline.setWriteFeed(feed);
+    return this;
+  }
+  setDataFeed(feed) {
+    var _a;
+    (0, import_node_assert9.default)(!this._dataFeed, "Data feed already set.");
+    this._dataFeed = feed;
+    (_a = this._dataPipeline.pipeline) == null ? void 0 : _a.setWriteFeed(feed);
+    return this;
+  }
+  /**
+  * Use for diagnostics.
+  */
+  getControlFeeds() {
+    return Array.from(this._controlPipeline.spaceState.feeds.values());
+  }
+  /**
+  * Use for diagnostics.
+  */
+  // getDataFeeds(): FeedInfo[] {
+  //   return this._dataPipeline?.getFeeds();
+  // }
+  async open() {
+    (0, import_log9.log)("opening...", {}, {
+      file: "space.ts",
+      line: 208,
+      scope: this,
+      callSite: (f, a) => f(...a)
+    });
+    if (this._isOpen) {
+      return;
+    }
+    await this._controlPipeline.start();
+    await this.protocol.start();
+    this._dataPipelineCredentialConsumer = this._controlPipeline.spaceState.registerProcessor(this._dataPipeline.createCredentialProcessor());
+    this._isOpen = true;
+    (0, import_log9.log)("opened", {}, {
+      file: "space.ts",
+      line: 222,
+      scope: this,
+      callSite: (f, a) => f(...a)
+    });
+  }
+  async close() {
+    var _a;
+    (0, import_log9.log)("closing...", {
+      key: this._key
+    }, {
+      file: "space.ts",
+      line: 227,
+      scope: this,
+      callSite: (f, a) => f(...a)
+    });
+    if (!this._isOpen) {
+      return;
+    }
+    await ((_a = this._dataPipelineCredentialConsumer) == null ? void 0 : _a.close());
+    this._dataPipelineCredentialConsumer = void 0;
+    await this._dataPipeline.close();
+    await this.protocol.stop();
+    await this._controlPipeline.stop();
+    this._isOpen = false;
+    (0, import_log9.log)("closed", {}, {
+      file: "space.ts",
+      line: 241,
+      scope: this,
+      callSite: (f, a) => f(...a)
+    });
+  }
+  async initializeDataPipeline() {
+    import_log9.log.info("initializeDataPipeline", {}, {
+      file: "space.ts",
+      line: 245,
+      scope: this,
+      callSite: (f, a) => f(...a)
+    });
+    (0, import_node_assert9.default)(this._isOpen, "Space must be open to initialize data pipeline.");
+    await this._dataPipeline.open();
+    await this._dataPipelineCredentialConsumer.open();
+  }
+};
+__decorate6([
+  import_log9.logInfo
+], Space.prototype, "key", null);
+__decorate6([
+  import_async7.synchronized
+], Space.prototype, "open", null);
+__decorate6([
+  import_async7.synchronized
+], Space.prototype, "close", null);
+Space = __decorate6([
+  (0, import_async7.trackLeaks)("open", "close")
+], Space);
+
+// packages/core/echo/echo-pipeline/src/space/space-manager.ts
+var import_async8 = require("@dxos/async");
+var import_debug5 = require("@dxos/debug");
+var import_keys5 = require("@dxos/keys");
+var import_log11 = require("@dxos/log");
+var import_protocols6 = require("@dxos/protocols");
+var import_util7 = require("@dxos/util");
+
+// packages/core/echo/echo-pipeline/src/space/space-protocol.ts
+var import_crypto3 = require("@dxos/crypto");
+var import_keys4 = require("@dxos/keys");
+var import_log10 = require("@dxos/log");
+var import_network_manager = require("@dxos/network-manager");
+var import_teleport2 = require("@dxos/teleport");
+var import_teleport_extension_replicator = require("@dxos/teleport-extension-replicator");
+var import_util6 = require("@dxos/util");
+var __decorate7 = function(decorators, target, key, desc) {
+  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+  if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
+    r = Reflect.decorate(decorators, target, key, desc);
+  else
+    for (var i = decorators.length - 1; i >= 0; i--)
+      if (d = decorators[i])
+        r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+  return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
 var MOCK_AUTH_PROVIDER = async (nonce) => Buffer.from("mock");
 var MOCK_AUTH_VERIFIER = async (nonce, credential) => true;
 var SpaceProtocol = class {
   constructor({ topic, swarmIdentity, networkManager, onSessionAuth }) {
     this._feeds = /* @__PURE__ */ new Set();
-    this._sessions = new import_util5.ComplexMap(import_keys4.PublicKey.hash);
+    this._sessions = new import_util6.ComplexMap(import_keys4.PublicKey.hash);
     this._networkManager = networkManager;
     this._swarmIdentity = swarmIdentity;
     this._onSessionAuth = onSessionAuth;
@@ -1289,7 +1674,7 @@ var SpaceProtocol = class {
   }
   // TODO(burdon): Create abstraction for Space (e.g., add keys and have provider).
   addFeed(feed) {
-    (0, import_log9.log)("addFeed", {
+    (0, import_log10.log)("addFeed", {
       key: feed.key
     }, {
       file: "space-protocol.ts",
@@ -1312,7 +1697,7 @@ var SpaceProtocol = class {
       maxPeers: 10,
       sampleSize: 20
     };
-    (0, import_log9.log)("starting...", {}, {
+    (0, import_log10.log)("starting...", {}, {
       file: "space-protocol.ts",
       line: 107,
       scope: this,
@@ -1325,7 +1710,7 @@ var SpaceProtocol = class {
       topology: new import_network_manager.MMSTTopology(topologyConfig),
       label: `Protocol swarm: ${this._topic}`
     });
-    (0, import_log9.log)("started", {}, {
+    (0, import_log10.log)("started", {}, {
       file: "space-protocol.ts",
       line: 116,
       scope: this,
@@ -1334,14 +1719,14 @@ var SpaceProtocol = class {
   }
   async stop() {
     if (this._connection) {
-      (0, import_log9.log)("stopping...", {}, {
+      (0, import_log10.log)("stopping...", {}, {
         file: "space-protocol.ts",
         line: 121,
         scope: this,
         callSite: (f, a) => f(...a)
       });
       await this._connection.close();
-      (0, import_log9.log)("stopped", {}, {
+      (0, import_log10.log)("stopped", {}, {
         file: "space-protocol.ts",
         line: 123,
         scope: this,
@@ -1364,11 +1749,11 @@ var SpaceProtocol = class {
     };
   }
 };
-__decorate6([
-  import_log9.logInfo
+__decorate7([
+  import_log10.logInfo
 ], SpaceProtocol.prototype, "_topic", void 0);
-__decorate6([
-  import_log9.logInfo
+__decorate7([
+  import_log10.logInfo
 ], SpaceProtocol.prototype, "_ownPeerKey", null);
 var AuthStatus;
 (function(AuthStatus2) {
@@ -1403,7 +1788,7 @@ var SpaceProtocolSession = class {
       onAuthSuccess: () => {
         var _a;
         this._authStatus = AuthStatus.SUCCESS;
-        (0, import_log9.log)("Peer authenticated", {}, {
+        (0, import_log10.log)("Peer authenticated", {}, {
           file: "space-protocol.ts",
           line: 206,
           scope: this,
@@ -1412,7 +1797,7 @@ var SpaceProtocolSession = class {
         (_a = this._onSessionAuth) == null ? void 0 : _a.call(this, this._teleport);
       },
       onAuthFailure: () => {
-        import_log9.log.warn("Auth failed", {}, {
+        import_log10.log.warn("Auth failed", {}, {
           file: "space-protocol.ts",
           line: 211,
           scope: this,
@@ -1427,15 +1812,15 @@ var SpaceProtocolSession = class {
     await this._teleport.close();
   }
 };
-__decorate6([
-  import_log9.logInfo
+__decorate7([
+  import_log10.logInfo
 ], SpaceProtocolSession.prototype, "_wireParams", void 0);
-__decorate6([
-  import_log9.logInfo
+__decorate7([
+  import_log10.logInfo
 ], SpaceProtocolSession.prototype, "authStatus", null);
 
 // packages/core/echo/echo-pipeline/src/space/space-manager.ts
-var __decorate7 = function(decorators, target, key, desc) {
+var __decorate8 = function(decorators, target, key, desc) {
   var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
   if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
     r = Reflect.decorate(decorators, target, key, desc);
@@ -1446,51 +1831,45 @@ var __decorate7 = function(decorators, target, key, desc) {
   return c > 3 && r && Object.defineProperty(target, key, r), r;
 };
 var SpaceManager = class SpaceManager2 {
-  constructor({ feedStore, networkManager }) {
-    this._spaces = new import_util6.ComplexMap(import_keys5.PublicKey.hash);
+  constructor({ feedStore, networkManager, modelFactory, metadataStore, snapshotStore }) {
+    this._spaces = new import_util7.ComplexMap(import_keys5.PublicKey.hash);
     this._instanceId = import_keys5.PublicKey.random().toHex();
     this._feedStore = feedStore;
     this._networkManager = networkManager;
+    this._modelFactory = modelFactory;
+    this._metadataStore = metadataStore;
+    this._snapshotStore = snapshotStore;
   }
   // TODO(burdon): Remove.
   get spaces() {
     return this._spaces;
   }
   async open() {
-    import_log10.log.trace("dxos.echo.space-manager", import_protocols6.trace.begin({
-      id: this._instanceId,
-      parentId: this._traceParent
-    }), {
-      file: "space-manager.ts",
-      line: 66,
-      scope: this,
-      callSite: (f, a) => f(...a)
-    });
   }
   async close() {
     await Promise.all([
       ...this._spaces.values()
     ].map((space) => space.close()));
-    import_log10.log.trace("dxos.echo.space-manager", import_protocols6.trace.end({
+  }
+  async constructSpace({ metadata, swarmIdentity, onNetworkConnection, memberKey }) {
+    var _a;
+    import_log11.log.trace("dxos.echo.space-manager.construct-space", import_protocols6.trace.begin({
       id: this._instanceId
     }), {
       file: "space-manager.ts",
-      line: 72,
+      line: 74,
       scope: this,
       callSite: (f, a) => f(...a)
     });
-  }
-  async constructSpace({ metadata, swarmIdentity, onNetworkConnection }) {
-    var _a;
-    (0, import_log10.log)("constructing space...", {
+    (0, import_log11.log)("constructing space...", {
       spaceKey: metadata.genesisFeedKey
     }, {
       file: "space-manager.ts",
-      line: 76,
+      line: 75,
       scope: this,
       callSite: (f, a) => f(...a)
     });
-    const genesisFeed = await this._feedStore.openFeed((_a = metadata.genesisFeedKey) != null ? _a : (0, import_debug4.failUndefined)());
+    const genesisFeed = await this._feedStore.openFeed((_a = metadata.genesisFeedKey) != null ? _a : (0, import_debug5.failUndefined)());
     const spaceKey = metadata.key;
     const protocol = new SpaceProtocol({
       topic: spaceKey,
@@ -1498,247 +1877,38 @@ var SpaceManager = class SpaceManager2 {
       networkManager: this._networkManager,
       onSessionAuth: onNetworkConnection
     });
+    const snapshotManager = new SnapshotManager(this._snapshotStore);
     const space = new Space({
       spaceKey,
       protocol,
       genesisFeed,
-      feedProvider: (feedKey) => this._feedStore.openFeed(feedKey)
+      feedProvider: (feedKey) => this._feedStore.openFeed(feedKey),
+      modelFactory: this._modelFactory,
+      metadataStore: this._metadataStore,
+      snapshotManager,
+      memberKey
     });
     this._spaces.set(space.key, space);
-    return space;
-  }
-};
-__decorate7([
-  import_async7.synchronized
-], SpaceManager.prototype, "open", null);
-__decorate7([
-  import_async7.synchronized
-], SpaceManager.prototype, "close", null);
-SpaceManager = __decorate7([
-  (0, import_async7.trackLeaks)("open", "close")
-], SpaceManager);
-
-// packages/core/echo/echo-pipeline/src/space/genesis.ts
-var import_credentials4 = require("@dxos/credentials");
-var import_debug5 = require("@dxos/debug");
-var import_credentials5 = require("@dxos/protocols/proto/dxos/halo/credentials");
-
-// packages/core/echo/echo-pipeline/src/space/data-pipeline.ts
-var import_node_assert9 = __toESM(require("node:assert"));
-var import_async8 = require("@dxos/async");
-var import_context3 = require("@dxos/context");
-var import_debug6 = require("@dxos/debug");
-var import_echo_db3 = require("@dxos/echo-db");
-var import_log11 = require("@dxos/log");
-var import_timeframe3 = require("@dxos/timeframe");
-var __decorate8 = function(decorators, target, key, desc) {
-  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
-  if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
-    r = Reflect.decorate(decorators, target, key, desc);
-  else
-    for (var i = decorators.length - 1; i >= 0; i--)
-      if (d = decorators[i])
-        r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
-  return c > 3 && r && Object.defineProperty(target, key, r), r;
-};
-var MESSAGES_PER_SNAPSHOT = 10;
-var AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL = 5e3;
-var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
-var DISABLE_SNAPSHOT_CACHE = true;
-var DataPipeline = class DataPipeline2 {
-  constructor(_params) {
-    this._params = _params;
-    this._ctx = new import_context3.Context();
-    this._lastAutomaticSnapshotTimeframe = new import_timeframe3.Timeframe();
-    this._isOpen = false;
-    this._lastTimeframeSaveTime = 0;
-    this._lastSnapshotSaveTime = 0;
-  }
-  get isOpen() {
-    return this._isOpen;
-  }
-  get pipelineState() {
-    var _a;
-    return (_a = this._pipeline) == null ? void 0 : _a.state;
-  }
-  get snapshotTimeframe() {
-    var _a;
-    return (_a = this._snapshot) == null ? void 0 : _a.timeframe;
-  }
-  getStartTimeframe() {
-    var _a;
-    return snapshotTimeframeToStartingTimeframe((_a = this.snapshotTimeframe) != null ? _a : new import_timeframe3.Timeframe());
-  }
-  setTargetTimeframe(timeframe) {
-    var _a;
-    this._targetTimeframe = timeframe;
-    (_a = this._pipeline) == null ? void 0 : _a.state.setTargetTimeframe(timeframe);
-  }
-  async open(spaceContext) {
-    var _a, _b, _c, _d;
-    if (this._isOpen) {
-      return;
-    }
-    this._spaceContext = spaceContext;
-    if (this._params.snapshotId && !DISABLE_SNAPSHOT_CACHE) {
-      this._snapshot = await this._params.snapshotManager.load(this._params.snapshotId);
-      this._lastAutomaticSnapshotTimeframe = (_b = (_a = this._snapshot) == null ? void 0 : _a.timeframe) != null ? _b : new import_timeframe3.Timeframe();
-    }
-    this._pipeline = await this._spaceContext.openPipeline(this.getStartTimeframe());
-    if (this._targetTimeframe) {
-      this._pipeline.state.setTargetTimeframe(this._targetTimeframe);
-    }
-    const feedWriter = createMappedFeedWriter((data) => ({
-      data
-    }), (_c = this._pipeline.writer) != null ? _c : (0, import_debug6.failUndefined)());
-    this.databaseBackend = new DatabaseHost(feedWriter, (_d = this._snapshot) == null ? void 0 : _d.database);
-    this._itemManager = new import_echo_db3.ItemManager(this._params.modelFactory);
-    await this.databaseBackend.open(this._itemManager, this._params.modelFactory);
-    (0, import_async8.scheduleTask)(this._ctx, async () => {
-      await this._consumePipeline();
-    });
-    this._isOpen = true;
-  }
-  async close() {
-    var _a, _b, _c;
-    if (!this._isOpen) {
-      return;
-    }
-    (0, import_log11.log)("close", {}, {
-      file: "data-pipeline.ts",
-      line: 145,
+    import_log11.log.trace("dxos.echo.space-manager.construct-space", import_protocols6.trace.end({
+      id: this._instanceId
+    }), {
+      file: "space-manager.ts",
+      line: 100,
       scope: this,
       callSite: (f, a) => f(...a)
     });
-    this._isOpen = false;
-    await this._ctx.dispose();
-    await ((_a = this._pipeline) == null ? void 0 : _a.stop());
-    try {
-      if (this._pipeline) {
-        await this._saveTargetTimeframe(this._pipeline.state.timeframe);
-        if (!DISABLE_SNAPSHOT_CACHE) {
-          await this._saveSnapshot(this._pipeline.state.timeframe);
-        }
-      }
-    } catch (err) {
-      import_log11.log.catch(err, {}, {
-        file: "data-pipeline.ts",
-        line: 160,
-        scope: this,
-        callSite: (f, a) => f(...a)
-      });
-    }
-    await ((_b = this.databaseBackend) == null ? void 0 : _b.close());
-    await ((_c = this._itemManager) == null ? void 0 : _c.destroy());
-    await this._params.snapshotManager.close();
-  }
-  async _consumePipeline() {
-    (0, import_node_assert9.default)(this._pipeline, "Pipeline is not initialized.");
-    for await (const msg of this._pipeline.consume()) {
-      const { feedKey, seq, data } = msg;
-      (0, import_log11.log)("processing message", {
-        msg
-      }, {
-        file: "data-pipeline.ts",
-        line: 172,
-        scope: this,
-        callSite: (f, a) => f(...a)
-      });
-      try {
-        if (data.payload.data) {
-          const feedInfo = this._params.feedInfoProvider(feedKey);
-          if (!feedInfo) {
-            import_log11.log.error("Could not find feed.", {
-              feedKey
-            }, {
-              file: "data-pipeline.ts",
-              line: 178,
-              scope: this,
-              callSite: (f, a) => f(...a)
-            });
-            continue;
-          }
-          await this.databaseBackend.echoProcessor({
-            batch: data.payload.data.batch,
-            meta: {
-              feedKey,
-              seq,
-              timeframe: data.timeframe,
-              memberKey: feedInfo.assertion.identityKey
-            }
-          });
-          await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
-        }
-      } catch (err) {
-        import_log11.log.catch(err, {}, {
-          file: "data-pipeline.ts",
-          line: 196,
-          scope: this,
-          callSite: (f, a) => f(...a)
-        });
-      }
-    }
-  }
-  _createSnapshot(timeframe) {
-    (0, import_node_assert9.default)(this.databaseBackend, "Database backend is not initialized.");
-    return {
-      spaceKey: this._params.spaceKey.asUint8Array(),
-      timeframe,
-      database: this.databaseBackend.createSnapshot()
-    };
-  }
-  async _saveSnapshot(timeframe) {
-    const snapshot = await this._createSnapshot(timeframe);
-    const snapshotKey = await this._params.snapshotManager.store(snapshot);
-    await this._params.metadataStore.setSpaceSnapshot(this._params.spaceKey, snapshotKey);
-    return snapshot;
-  }
-  async _saveTargetTimeframe(timeframe) {
-    var _a;
-    const newTimeframe = import_timeframe3.Timeframe.merge((_a = this._targetTimeframe) != null ? _a : new import_timeframe3.Timeframe(), timeframe);
-    await this._params.metadataStore.setSpaceLatestTimeframe(this._params.spaceKey, newTimeframe);
-    this._targetTimeframe = newTimeframe;
-  }
-  async _noteTargetStateIfNeeded(timeframe) {
-    var _a;
-    if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL) {
-      this._lastTimeframeSaveTime = Date.now();
-      await this._saveTargetTimeframe(timeframe);
-    }
-    if (!DISABLE_SNAPSHOT_CACHE && Date.now() - this._lastSnapshotSaveTime > AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL && timeframe.totalMessages() - this._lastAutomaticSnapshotTimeframe.totalMessages() > MESSAGES_PER_SNAPSHOT) {
-      this._lastSnapshotSaveTime = Date.now();
-      const snapshot = await this._saveSnapshot(timeframe);
-      this._lastAutomaticSnapshotTimeframe = (_a = snapshot.timeframe) != null ? _a : (0, import_debug6.failUndefined)();
-      (0, import_log11.log)("save", {
-        snapshot
-      }, {
-        file: "data-pipeline.ts",
-        line: 239,
-        scope: this,
-        callSite: (f, a) => f(...a)
-      });
-    }
-  }
-  async waitUntilTimeframe(timeframe) {
-    (0, import_node_assert9.default)(this._pipeline, "Pipeline is not initialized.");
-    await this._pipeline.state.waitUntilTimeframe(timeframe);
+    return space;
   }
 };
 __decorate8([
   import_async8.synchronized
-], DataPipeline.prototype, "open", null);
+], SpaceManager.prototype, "open", null);
 __decorate8([
   import_async8.synchronized
-], DataPipeline.prototype, "close", null);
-DataPipeline = __decorate8([
+], SpaceManager.prototype, "close", null);
+SpaceManager = __decorate8([
   (0, import_async8.trackLeaks)("open", "close")
-], DataPipeline);
-var snapshotTimeframeToStartingTimeframe = (snapshotTimeframe) => {
-  return snapshotTimeframe.map(([key, seq]) => [
-    key,
-    seq + 1
-  ]);
-};
+], SpaceManager);
 
 // packages/core/echo/echo-pipeline/src/testing/test-feed-builder.ts
 var import_testing = require("@dxos/feed-store/testing");
@@ -1765,7 +1935,7 @@ var WebsocketNetworkManagerProvider = (signalUrl) => () => new import_network_ma
 });
 var TestAgentBuilder = class {
   constructor({ storage, networkManagerProvider } = {}) {
-    this._agents = new import_util7.ComplexMap(import_keys6.PublicKey.hash);
+    this._agents = new import_util8.ComplexMap(import_keys6.PublicKey.hash);
     this._storage = storage != null ? storage : (0, import_random_access_storage.createStorage)({
       type: import_random_access_storage.StorageType.RAM
     });
@@ -1790,12 +1960,22 @@ var TestAgentBuilder = class {
   }
 };
 var TestAgent = class {
+  get metadataStore() {
+    var _a;
+    return (_a = this._metadataStore) != null ? _a : this._metadataStore = new MetadataStore(this.storage.createDirectory("metadata"));
+  }
+  get snapshotStore() {
+    var _a;
+    return (_a = this._snapshotStore) != null ? _a : this._snapshotStore = new SnapshotStore(this.storage.createDirectory("snapshots"));
+  }
   constructor(_networkManagerProvider, _feedBuilder, identityKey, deviceKey) {
     this._networkManagerProvider = _networkManagerProvider;
     this._feedBuilder = _feedBuilder;
     this.identityKey = identityKey;
     this.deviceKey = deviceKey;
-    this._spaces = new import_util7.ComplexMap(import_keys6.PublicKey.hash);
+    this._spaces = new import_util8.ComplexMap(import_keys6.PublicKey.hash);
+    this.modelFactory = new import_model_factory.ModelFactory().registerModel(import_document_model.DocumentModel);
+    this.storage = this._feedBuilder.storage;
     this.keyring = this._feedBuilder.keyring;
     this.feedStore = this._feedBuilder.createFeedStore();
   }
@@ -1810,60 +1990,60 @@ var TestAgent = class {
   getSpace(spaceKey) {
     return this._spaces.get(spaceKey);
   }
-  createSpaceManager() {
-    return new SpaceManager({
-      feedStore: this._feedBuilder.createFeedStore(),
-      networkManager: this._networkManagerProvider()
+  get spaceManager() {
+    var _a;
+    return (_a = this._spaceManager) != null ? _a : this._spaceManager = new SpaceManager({
+      feedStore: this.feedStore,
+      networkManager: this._networkManagerProvider(),
+      modelFactory: this.modelFactory,
+      metadataStore: this.metadataStore,
+      snapshotStore: this.snapshotStore
     });
   }
-  async createSpace(identityKey = this.identityKey, spaceKey, genesisKey) {
+  async createSpace(identityKey = this.identityKey, spaceKey, genesisKey, dataKey) {
+    let saveMetadata = false;
     if (!spaceKey) {
+      saveMetadata = true;
       spaceKey = await this.keyring.createKey();
     }
-    const dataFeedKey = await this.keyring.createKey();
-    const dataFeed = await this.feedStore.openFeed(dataFeedKey, {
+    if (!genesisKey) {
+      genesisKey = await this.keyring.createKey();
+    }
+    const controlFeed = await this.feedStore.openFeed(genesisKey, {
       writable: true
     });
-    const controlFeedKey = await this.keyring.createKey();
-    const controlFeed = await this.feedStore.openFeed(controlFeedKey, {
+    const dataFeed = await this.feedStore.openFeed(dataKey != null ? dataKey : await this.keyring.createKey(), {
       writable: true
     });
-    const genesisFeed = genesisKey ? await this.feedStore.openFeed(genesisKey) : controlFeed;
-    const snapshotManager = new SnapshotManager(new SnapshotStore((0, import_random_access_storage.createStorage)().createDirectory("snapshots")));
-    const metadataStore = new MetadataStore((0, import_random_access_storage.createStorage)().createDirectory("metadata"));
-    await metadataStore.addSpace({
-      key: spaceKey
-    });
-    const dataPipelineController = new DataPipeline({
-      modelFactory: new import_model_factory.ModelFactory().registerModel(import_document_model.DocumentModel),
-      metadataStore,
-      snapshotManager,
+    const metadata = {
+      key: spaceKey,
+      genesisFeedKey: genesisKey,
+      controlFeedKey: controlFeed.key,
+      dataFeedKey: dataFeed.key
+    };
+    if (saveMetadata) {
+      await this.metadataStore.addSpace(metadata);
+    }
+    await this.spaceManager.open();
+    const space = await this.spaceManager.constructSpace({
+      metadata,
+      swarmIdentity: {
+        peerKey: this.deviceKey,
+        credentialProvider: MOCK_AUTH_PROVIDER,
+        credentialAuthenticator: MOCK_AUTH_VERIFIER
+      },
       memberKey: identityKey,
-      spaceKey,
-      feedInfoProvider: (feedKey) => space.spaceState.feeds.get(feedKey),
-      snapshotId: void 0
-    });
-    const space = new Space({
-      spaceKey,
-      protocol: this.createSpaceProtocol(spaceKey),
-      genesisFeed,
-      feedProvider: (feedKey) => this.feedStore.openFeed(feedKey)
-    }).setControlFeed(controlFeed).setDataFeed(dataFeed);
-    await space.open();
-    await dataPipelineController.open({
-      openPipeline: async (start) => {
-        const pipeline = await space.createDataPipeline({
-          start
-        });
-        await pipeline.start();
-        return pipeline;
+      onNetworkConnection: (session) => {
+        session.addExtension("dxos.mesh.teleport.gossip", this.createGossip().createExtension({
+          remotePeerId: session.remotePeerId
+        }));
       }
     });
+    space.setControlFeed(controlFeed);
+    space.setDataFeed(dataFeed);
+    await space.open();
     this._spaces.set(spaceKey, space);
-    return [
-      space,
-      dataPipelineController
-    ];
+    return space;
   }
   createSpaceProtocol(topic, gossip) {
     return new SpaceProtocol({
@@ -1894,6 +2074,21 @@ var TestAgent = class {
       gossip: gossip != null ? gossip : this.createGossip()
     });
   }
+  async spaceGenesis(space) {
+    const generator = new import_credentials5.CredentialGenerator(this.keyring, this.identityKey, this.deviceKey);
+    const credentials = [
+      ...await generator.createSpaceGenesis(space.key, space.controlFeedKey),
+      await generator.createFeedAdmission(space.key, space.dataFeedKey, import_credentials6.AdmittedFeed.Designation.DATA),
+      await generator.createEpochCredential(space.key)
+    ];
+    for (const credential of credentials) {
+      await space.controlPipeline.writer.write({
+        credential: {
+          credential
+        }
+      });
+    }
+  }
 };
 
 // packages/core/echo/echo-pipeline/src/testing/util.ts
@@ -1906,7 +2101,7 @@ var import_model_factory2 = require("@dxos/model-factory");
 var import_timeframe4 = require("@dxos/timeframe");
 var createMemoryDatabase = async (modelFactory) => {
   const feed = new import_testing2.MockFeedWriter();
-  const backend = new DatabaseHost(feed, void 0);
+  const backend = new DatabaseHost(feed);
   feed.written.on(([data, meta]) => backend.echoProcessor({
     batch: data.batch,
     meta: {
@@ -1943,7 +2138,7 @@ var createRemoteDatabaseFromDataServiceHost = async (modelFactory, dataServiceHo
 var testLocalDatabase = async (create, check = create) => {
   var _a;
   const objectId = import_keys7.PublicKey.random().toHex();
-  await ((_a = create.databaseBackend.getWriteStream()) == null ? void 0 : _a.write({
+  await ((_a = create.databaseHost.getWriteStream()) == null ? void 0 : _a.write({
     batch: {
       objects: [
         {
@@ -1955,7 +2150,7 @@ var testLocalDatabase = async (create, check = create) => {
       ]
     }
   }));
-  await (0, import_async9.asyncTimeout)(check.databaseBackend._itemDemuxer.mutation.waitForCondition(() => check._itemManager.entities.has(objectId)), 500);
+  await (0, import_async9.asyncTimeout)(check.databaseHost._itemDemuxer.mutation.waitForCondition(() => check.itemManager.entities.has(objectId)), 500);
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {