@dxos/echo-pipeline 0.1.53-next.63b0547 → 0.1.53-next.e9dc2bc

package/dist/lib/node/index.cjs

@@ -120,6 +120,7 @@ var import_async = require("@dxos/async");
 var import_errors = require("@dxos/errors");
 var import_log = require("@dxos/log");
 var import_protocols = require("@dxos/protocols");
+var import_services = require("@dxos/protocols/proto/dxos/client/services");
 var import_util = require("@dxos/util");
 function _ts_decorate(decorators, target, key, desc) {
   var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
@@ -138,6 +139,7 @@ var emptyEchoMetadata = () => ({
   created: new Date(),
   updated: new Date()
 });
+var EchoMetadata = import_protocols.schema.getCodecForType("dxos.echo.metadata.EchoMetadata");
 var MetadataStore = class {
   // prettier-ignore
   constructor(_directory) {
@@ -164,6 +166,7 @@ var MetadataStore = class {
   * Loads metadata from persistent storage.
   */
   async load() {
+    var _a;
     const file = this._directory.getOrCreateFile("EchoMetadata");
     try {
       const { size: fileLength } = await file.stat();
@@ -177,7 +180,7 @@ var MetadataStore = class {
         checksum
       }, {
         F: __dxlog_file,
-        L: 69,
+        L: 72,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -189,13 +192,17 @@ var MetadataStore = class {
       if (calculatedChecksum !== checksum) {
         throw new import_errors.DataCorruptionError("Metadata checksum is invalid.");
       }
-      this._metadata = import_protocols.schema.getCodecForType("dxos.echo.metadata.EchoMetadata").decode(data);
+      this._metadata = EchoMetadata.decode(data);
+      (_a = this._metadata.spaces) == null ? void 0 : _a.forEach((space) => {
+        var _a2;
+        (_a2 = space.state) != null ? _a2 : space.state = import_services.SpaceState.ACTIVE;
+      });
     } catch (err) {
       import_log.log.error("failed to load metadata", {
         err
       }, {
         F: __dxlog_file,
-        L: 84,
+        L: 92,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -215,7 +222,7 @@ var MetadataStore = class {
     this.update.emit(data);
     const file = this._directory.getOrCreateFile("EchoMetadata");
     try {
-      const encoded = (0, import_util.arrayToBuffer)(import_protocols.schema.getCodecForType("dxos.echo.metadata.EchoMetadata").encode(data));
+      const encoded = (0, import_util.arrayToBuffer)(EchoMetadata.encode(data));
       const checksum = import_crc_32.default.buf(encoded);
       const result = Buffer.alloc(8 + encoded.length);
       result.writeInt32LE(encoded.length, 0);
@@ -227,7 +234,7 @@ var MetadataStore = class {
         checksum
       }, {
         F: __dxlog_file,
-        L: 116,
+        L: 124,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -250,7 +257,7 @@ var MetadataStore = class {
   async clear() {
     (0, import_log.log)("clearing all metadata", void 0, {
       F: __dxlog_file,
-      L: 137,
+      L: 145,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -289,6 +296,10 @@ var MetadataStore = class {
     space.dataFeedKey = dataFeedKey;
     await this._save();
   }
+  async setSpaceState(spaceKey, state) {
+    this._getSpace(spaceKey).state = state;
+    await this._save();
+  }
 };
 _ts_decorate([
   import_async.synchronized
@@ -462,6 +473,7 @@ var PipelineState = class {
   constructor(_feeds, _timeframeClock) {
     this._feeds = _feeds;
     this._timeframeClock = _timeframeClock;
+    this._ctx = new import_context.Context();
     this.timeframeUpdate = this._timeframeClock.update;
     this.stalled = new import_async3.Event();
     this._startTimeframe = new import_timeframe2.Timeframe();
@@ -513,7 +525,7 @@ var PipelineState = class {
       target: this.targetTimeframe
     }, {
       F: __dxlog_file4,
-      L: 118,
+      L: 126,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -529,6 +541,7 @@ var PipelineState = class {
     if (timeout) {
       return Promise.race([
         (0, import_context.rejectOnDispose)(ctx),
+        (0, import_context.rejectOnDispose)(this._ctx),
         this._reachedTargetPromise.then(() => {
           done = true;
         }),
@@ -543,7 +556,7 @@ var PipelineState = class {
             dependencies: import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe)
           }, {
             F: __dxlog_file4,
-            L: 144,
+            L: 153,
             S: this,
             C: (f, a) => f(...a)
           });
@@ -601,7 +614,7 @@ var Pipeline = class {
   async start() {
     (0, import_log4.log)("starting...", {}, {
       F: __dxlog_file4,
-      L: 264,
+      L: 273,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -609,7 +622,7 @@ var Pipeline = class {
     await this._feedSetIterator.open();
     (0, import_log4.log)("started", void 0, {
       F: __dxlog_file4,
-      L: 267,
+      L: 276,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -619,16 +632,19 @@ var Pipeline = class {
     var _a;
     (0, import_log4.log)("stopping...", {}, {
       F: __dxlog_file4,
-      L: 273,
+      L: 282,
       S: this,
       C: (f, a) => f(...a)
     });
     this._isStopping = true;
     await ((_a = this._feedSetIterator) == null ? void 0 : _a.close());
     await this._processingTrigger.wait();
+    await this._state._ctx.dispose();
+    this._state._ctx = new import_context.Context();
+    this._state._reachedTargetPromise = void 0;
     (0, import_log4.log)("stopped", void 0, {
       F: __dxlog_file4,
-      L: 277,
+      L: 289,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -705,7 +721,7 @@ var Pipeline = class {
     feed.undownload({
       callback: () => (0, import_log4.log)("Undownloaded", void 0, {
         F: __dxlog_file4,
-        L: 371,
+        L: 383,
         S: this,
         C: (f, a) => f(...a)
       })
@@ -718,7 +734,7 @@ var Pipeline = class {
         err
       }, {
         F: __dxlog_file4,
-        L: 374,
+        L: 386,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -732,7 +748,7 @@ var Pipeline = class {
     this._feedSetIterator.stalled.on((iterator) => {
       import_log4.log.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, void 0, {
         F: __dxlog_file4,
-        L: 384,
+        L: 396,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -1001,7 +1017,7 @@ var ControlPipeline = class {
 // packages/core/echo/echo-pipeline/src/space/data-pipeline.ts
 var import_tiny_invariant8 = __toESM(require("tiny-invariant"));
 var import_async6 = require("@dxos/async");
-var import_context4 = require("@dxos/context");
+var import_context5 = require("@dxos/context");
 var import_credentials3 = require("@dxos/credentials");
 var import_echo_db3 = require("@dxos/echo-db");
 var import_errors2 = require("@dxos/errors");
@@ -1011,6 +1027,7 @@ var import_timeframe4 = require("@dxos/timeframe");
 // packages/core/echo/echo-pipeline/src/dbhost/data-service-host.ts
 var import_tiny_invariant6 = __toESM(require("tiny-invariant"));
 var import_codec_protobuf = require("@dxos/codec-protobuf");
+var import_context3 = require("@dxos/context");
 var import_echo_db = require("@dxos/echo-db");
 var import_log7 = require("@dxos/log");
 var import_util4 = require("@dxos/util");
@@ -1020,13 +1037,20 @@ var DataServiceHost = class {
     this._itemManager = _itemManager;
     this._itemDemuxer = _itemDemuxer;
     this._writeStream = _writeStream;
+    this._ctx = new import_context3.Context();
     this._clientTagMap = new import_util4.ComplexMap(([feedKey, seq]) => `${feedKey.toHex()}:${seq}`);
   }
+  async open() {
+  }
+  async close() {
+    await this._ctx.dispose();
+  }
   /**
   * Real-time subscription to data objects in a space.
   */
   subscribe() {
-    return new import_codec_protobuf.Stream(({ next, ctx }) => {
+    return new import_codec_protobuf.Stream(({ next, close, ctx }) => {
+      ctx.onDispose(this._ctx.onDispose(close));
       const objects = Array.from(this._itemManager.entities.values()).map((entity) => entity.createSnapshot());
       next({
         batch: {
@@ -1042,7 +1066,7 @@ var DataServiceHost = class {
           meta
         }, {
           F: __dxlog_file7,
-          L: 51,
+          L: 63,
           S: this,
           C: (f, a) => f(...a)
         });
@@ -1069,13 +1093,14 @@ var DataServiceHost = class {
   }
   async write(request) {
     var _a, _b;
+    (0, import_tiny_invariant6.default)(!this._ctx.disposed, "Cannot write to closed DataServiceHost");
     (0, import_tiny_invariant6.default)(this._writeStream, "Cannot write mutations in readonly mode");
     (0, import_log7.log)("write", {
       clientTag: request.clientTag,
       objectCount: (_b = (_a = request.batch.objects) == null ? void 0 : _a.length) != null ? _b : 0
     }, {
       F: __dxlog_file7,
-      L: 79,
+      L: 92,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -1089,7 +1114,7 @@ var DataServiceHost = class {
             seq: receipt2.seq
           }, {
             F: __dxlog_file7,
-            L: 88,
+            L: 101,
             S: this,
             C: (f, a) => f(...a)
           });
@@ -1156,7 +1181,7 @@ var DatabaseHost = class {
 
 // packages/core/echo/echo-pipeline/src/dbhost/snapshot-manager.ts
 var import_async5 = require("@dxos/async");
-var import_context3 = require("@dxos/context");
+var import_context4 = require("@dxos/context");
 var import_debug3 = require("@dxos/debug");
 var import_keys3 = require("@dxos/keys");
 var import_protocols4 = require("@dxos/protocols");
@@ -1171,6 +1196,7 @@ function _ts_decorate4(decorators, target, key, desc) {
         r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
   return c > 3 && r && Object.defineProperty(target, key, r), r;
 }
+var SpaceSnapshot = import_protocols4.schema.getCodecForType("dxos.echo.snapshot.SpaceSnapshot");
 var SnapshotManager = class SnapshotManager2 {
   // prettier-ignore
   constructor(_snapshotStore, _blobStore, _blobSync) {
@@ -1180,7 +1206,7 @@ var SnapshotManager = class SnapshotManager2 {
   }
   async _getBlob(blobId) {
     const blob = await this._blobStore.get(blobId);
-    return import_protocols4.schema.getCodecForType("dxos.echo.snapshot.SpaceSnapshot").decode(blob);
+    return SpaceSnapshot.decode(blob);
   }
   async load(ctx, id) {
     const blobId = import_keys3.PublicKey.fromHex(id).asUint8Array();
@@ -1188,7 +1214,7 @@ var SnapshotManager = class SnapshotManager2 {
     if (blobMeta && blobMeta.state === import_blob.BlobMeta.State.FULLY_PRESENT) {
       return this._getBlob(blobId);
     }
-    const fallbackStore = await (0, import_context3.cancelWithContext)(ctx, this._snapshotStore.loadSnapshot(id));
+    const fallbackStore = await (0, import_context4.cancelWithContext)(ctx, this._snapshotStore.loadSnapshot(id));
     if (fallbackStore) {
       return fallbackStore;
     }
@@ -1196,7 +1222,7 @@ var SnapshotManager = class SnapshotManager2 {
     return this._getBlob(blobId);
   }
   async store(snapshot) {
-    const { id } = await this._blobStore.set(import_protocols4.schema.getCodecForType("dxos.echo.snapshot.SpaceSnapshot").encode(snapshot));
+    const { id } = await this._blobStore.set(SpaceSnapshot.encode(snapshot));
     await this._blobSync.notifyBlobAdded(id);
     return import_keys3.PublicKey.from(id).toHex();
   }
@@ -1211,13 +1237,14 @@ SnapshotManager = _ts_decorate4([
 // packages/core/echo/echo-pipeline/src/dbhost/snapshot-store.ts
 var import_crypto2 = require("@dxos/crypto");
 var import_protocols5 = require("@dxos/protocols");
+var SpaceSnapshot2 = import_protocols5.schema.getCodecForType("dxos.echo.snapshot.SpaceSnapshot");
 var SnapshotStore = class {
   // prettier-ignore
   constructor(_directory) {
     this._directory = _directory;
   }
   async saveSnapshot(snapshot) {
-    const encoded = import_protocols5.schema.getCodecForType("dxos.echo.snapshot.SpaceSnapshot").encode(snapshot);
+    const encoded = SpaceSnapshot2.encode(snapshot);
     const key = await import_crypto2.subtleCrypto.digest("SHA-256", encoded);
     const keyString = Buffer.from(key).toString("hex");
     const file = await this._directory.getOrCreateFile(keyString);
@@ -1236,7 +1263,7 @@ var SnapshotStore = class {
         return void 0;
       }
       const buffer = await file.read(0, size);
-      return import_protocols5.schema.getCodecForType("dxos.echo.snapshot.SpaceSnapshot").decode(buffer);
+      return SpaceSnapshot2.decode(buffer);
     } finally {
       await file.close();
     }
@@ -1267,7 +1294,7 @@ var DataServiceSubscriptions = class {
   clear() {
     this._spaces.clear();
   }
-  registerSpace(spaceKey, host) {
+  async registerSpace(spaceKey, host) {
     (0, import_log8.log)("Registering space", {
       spaceKey
     }, {
@@ -1276,17 +1303,21 @@ var DataServiceSubscriptions = class {
       S: this,
       C: (f, a) => f(...a)
     });
+    (0, import_tiny_invariant7.default)(!this._spaces.has(spaceKey));
+    await host.open();
     this._spaces.set(spaceKey, host);
   }
-  unregisterSpace(spaceKey) {
+  async unregisterSpace(spaceKey) {
     (0, import_log8.log)("Unregistering space", {
       spaceKey
     }, {
       F: __dxlog_file8,
-      L: 36,
+      L: 38,
       S: this,
       C: (f, a) => f(...a)
     });
+    const host = this._spaces.get(spaceKey);
+    await (host == null ? void 0 : host.close());
     this._spaces.delete(spaceKey);
   }
   getDataService(spaceKey) {
@@ -1330,11 +1361,15 @@ var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL2 = 500;
 var DataPipeline = class DataPipeline2 {
   constructor(_params) {
     this._params = _params;
-    this._ctx = new import_context4.Context();
+    this._ctx = new import_context5.Context();
+    this._pipeline = void 0;
+    this._targetTimeframe = void 0;
     this._lastAutomaticSnapshotTimeframe = new import_timeframe4.Timeframe();
     this._isOpen = false;
     this._lastTimeframeSaveTime = 0;
     this._lastSnapshotSaveTime = 0;
+    this.currentEpoch = void 0;
+    this.appliedEpoch = void 0;
     this._lastProcessedEpoch = -1;
     this.onNewEpoch = new import_async6.Event();
   }
@@ -1353,18 +1388,14 @@ var DataPipeline = class DataPipeline2 {
     this._targetTimeframe = timeframe;
     (_a = this._pipeline) == null ? void 0 : _a.state.setTargetTimeframe(timeframe);
   }
-  createCredentialProcessor() {
-    return {
-      process: async (credential) => {
-        if (!(0, import_credentials3.checkCredentialType)(credential, "dxos.halo.credentials.Epoch")) {
-          return;
-        }
-        this.currentEpoch = credential;
-        if (this._isOpen) {
-          await this._processEpochInSeparateTask(credential);
-        }
-      }
-    };
+  async processCredential(credential) {
+    if (!(0, import_credentials3.checkCredentialType)(credential, "dxos.halo.credentials.Epoch")) {
+      return;
+    }
+    this.currentEpoch = credential;
+    if (this._isOpen) {
+      await this._processEpochInSeparateTask(credential);
+    }
   }
   async open() {
     if (this._isOpen) {
@@ -1401,7 +1432,7 @@ var DataPipeline = class DataPipeline2 {
     }
     (0, import_log9.log)("close", void 0, {
       F: __dxlog_file9,
-      L: 174,
+      L: 171,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -1416,13 +1447,21 @@ var DataPipeline = class DataPipeline2 {
     } catch (err) {
       import_log9.log.catch(err, void 0, {
         F: __dxlog_file9,
-        L: 187,
+        L: 184,
         S: this,
         C: (f, a) => f(...a)
       });
     }
     await ((_b = this.databaseHost) == null ? void 0 : _b.close());
     await ((_c = this.itemManager) == null ? void 0 : _c.destroy());
+    this._ctx = new import_context5.Context();
+    this._pipeline = void 0;
+    this._targetTimeframe = void 0;
+    this._lastAutomaticSnapshotTimeframe = new import_timeframe4.Timeframe();
+    this.currentEpoch = void 0;
+    this.appliedEpoch = void 0;
+    this._lastProcessedEpoch = -1;
+    this._epochCtx = void 0;
   }
   async _consumePipeline() {
     if (this.currentEpoch) {
@@ -1438,7 +1477,7 @@ var DataPipeline = class DataPipeline2 {
         seq
       }, {
         F: __dxlog_file9,
-        L: 204,
+        L: 210,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -1450,7 +1489,7 @@ var DataPipeline = class DataPipeline2 {
               feedKey
             }, {
               F: __dxlog_file9,
-              L: 210,
+              L: 216,
               S: this,
               C: (f, a) => f(...a)
             });
@@ -1471,7 +1510,7 @@ var DataPipeline = class DataPipeline2 {
             spaceKey: this._params.spaceKey.toHex()
           }, {
             F: __dxlog_file9,
-            L: 224,
+            L: 230,
             S: this,
             C: (f, a) => f(...a)
           });
@@ -1480,7 +1519,7 @@ var DataPipeline = class DataPipeline2 {
       } catch (err) {
         import_log9.log.catch(err, void 0, {
           F: __dxlog_file9,
-          L: 234,
+          L: 240,
           S: this,
           C: (f, a) => f(...a)
         });
@@ -1515,7 +1554,7 @@ var DataPipeline = class DataPipeline2 {
     } catch (err) {
       import_log9.log.warn("Failed to cache properties", err, {
         F: __dxlog_file9,
-        L: 269,
+        L: 275,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -1537,19 +1576,19 @@ var DataPipeline = class DataPipeline2 {
       return;
     }
     await ((_a = this._epochCtx) == null ? void 0 : _a.dispose());
-    const ctx = new import_context4.Context({
+    const ctx = new import_context5.Context({
       onError: (err) => {
         if (err instanceof import_errors2.CancelledError) {
           (0, import_log9.log)("Epoch processing cancelled.", void 0, {
             F: __dxlog_file9,
-            L: 301,
+            L: 307,
             S: this,
             C: (f, a) => f(...a)
           });
         } else {
           import_log9.log.catch(err, void 0, {
             F: __dxlog_file9,
-            L: 303,
+            L: 309,
             S: this,
             C: (f, a) => f(...a)
           });
@@ -1565,7 +1604,7 @@ var DataPipeline = class DataPipeline2 {
         epoch
       }, {
         F: __dxlog_file9,
-        L: 313,
+        L: 319,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -1582,7 +1621,7 @@ var DataPipeline = class DataPipeline2 {
       epoch
     }, {
       F: __dxlog_file9,
-      L: 327,
+      L: 333,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -1592,7 +1631,7 @@ var DataPipeline = class DataPipeline2 {
     }
     (0, import_log9.log)("restarting pipeline for epoch", void 0, {
       F: __dxlog_file9,
-      L: 335,
+      L: 341,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -1656,8 +1695,6 @@ var Space = class Space2 {
     this._addFeedLock = new import_async7.Lock();
     this.onCredentialProcessed = new import_util6.Callback();
     this.stateUpdate = new import_async7.Event();
-    // Processes epoch credentials.
-    this._dataPipelineCredentialConsumer = void 0;
     this._isOpen = false;
     (0, import_tiny_invariant9.default)(params.spaceKey && params.feedProvider);
     this._key = params.spaceKey;
@@ -1695,7 +1732,7 @@ var Space = class Space2 {
         credential
       }, {
         F: __dxlog_file10,
-        L: 110,
+        L: 107,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -1787,7 +1824,7 @@ var Space = class Space2 {
   async open() {
     (0, import_log10.log)("opening...", void 0, {
       F: __dxlog_file10,
-      L: 212,
+      L: 209,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -1796,38 +1833,35 @@ var Space = class Space2 {
     }
     await this._controlPipeline.start();
     await this.protocol.start();
-    this._dataPipelineCredentialConsumer = this._controlPipeline.spaceState.registerProcessor(this._dataPipeline.createCredentialProcessor());
-    await this._dataPipelineCredentialConsumer.open();
+    await this._controlPipeline.spaceState.addCredentialProcessor(this._dataPipeline);
     this._isOpen = true;
     (0, import_log10.log)("opened", void 0, {
       F: __dxlog_file10,
-      L: 228,
+      L: 220,
       S: this,
       C: (f, a) => f(...a)
     });
   }
   async close() {
-    var _a;
     (0, import_log10.log)("closing...", {
       key: this._key
     }, {
       F: __dxlog_file10,
-      L: 233,
+      L: 225,
       S: this,
       C: (f, a) => f(...a)
     });
     if (!this._isOpen) {
       return;
     }
-    await ((_a = this._dataPipelineCredentialConsumer) == null ? void 0 : _a.close());
-    this._dataPipelineCredentialConsumer = void 0;
+    await this._controlPipeline.spaceState.removeCredentialProcessor(this._dataPipeline);
     await this._dataPipeline.close();
     await this.protocol.stop();
     await this._controlPipeline.stop();
     this._isOpen = false;
     (0, import_log10.log)("closed", void 0, {
       F: __dxlog_file10,
-      L: 247,
+      L: 238,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -1835,7 +1869,7 @@ var Space = class Space2 {
   async initializeDataPipeline() {
     (0, import_log10.log)("initializeDataPipeline", void 0, {
       F: __dxlog_file10,
-      L: 252,
+      L: 243,
       S: this,
       C: (f, a) => f(...a)
     });