@dxos/echo-pipeline 0.3.3-main.f2ca85a → 0.3.3-main.f5c635b

package/dist/lib/node/index.cjs

@@ -91,6 +91,7 @@ var createMappedFeedWriter = (mapper, writer) => {
 };
 
 // packages/core/echo/echo-pipeline/src/db-host/data-service-host.ts
+var import_async = require("@dxos/async");
 var import_codec_protobuf = require("@dxos/codec-protobuf");
 var import_context = require("@dxos/context");
 var import_echo_db = require("@dxos/echo-db");
@@ -99,12 +100,14 @@ var import_log = require("@dxos/log");
 var import_service = require("@dxos/protocols/proto/dxos/echo/service");
 var import_util = require("@dxos/util");
 var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/data-service-host.ts";
+var MUTATION_LIMIT_PER_OBJECT = 10;
 var DataServiceHost = class {
-  constructor(_itemManager, _itemDemuxer, _flush, _writeStream) {
+  constructor(_itemManager, _itemDemuxer, _flush, _writeStream, _opts = {}) {
     this._itemManager = _itemManager;
     this._itemDemuxer = _itemDemuxer;
     this._flush = _flush;
     this._writeStream = _writeStream;
+    this._opts = _opts;
     this._ctx = new import_context.Context();
     this._clientTagMap = new import_util.ComplexMap(([feedKey, seq]) => `${feedKey.toHex()}:${seq}`);
   }
@@ -113,6 +116,9 @@ var DataServiceHost = class {
   async close() {
     await this._ctx.dispose();
   }
+  get _deferEvents() {
+    return this._opts.deferEvents ?? true;
+  }
   /**
   * Real-time subscription to data objects in a space.
   */
@@ -125,7 +131,43 @@ var DataServiceHost = class {
           objects
         }
       });
+      const updateScheduler = new import_async.UpdateScheduler(ctx, async () => {
+        flushPendingUpdate();
+      }, {
+        maxFrequency: 10
+      });
+      const pendingUpdates = [];
+      const mutationsPerObject = /* @__PURE__ */ new Map();
+      const clearPendingUpdates = () => {
+        pendingUpdates.length = 0;
+        mutationsPerObject.clear();
+      };
+      const flushPendingUpdate = () => {
+        const stagedEvents = [];
+        const objectsWithSnapshots = /* @__PURE__ */ new Set();
+        for (const [id, count] of mutationsPerObject) {
+          if (count >= MUTATION_LIMIT_PER_OBJECT) {
+            objectsWithSnapshots.add(id);
+            const entity = this._itemManager.entities.get(id);
+            if (entity) {
+              stagedEvents.push(entity.createSnapshot());
+            }
+          }
+        }
+        for (const obj of pendingUpdates) {
+          if (!objectsWithSnapshots.has(obj.objectId)) {
+            stagedEvents.push(obj);
+          }
+        }
+        next({
+          batch: {
+            objects: stagedEvents
+          }
+        });
+        clearPendingUpdates();
+      };
       this._itemDemuxer.snapshot.on(ctx, (snapshot) => {
+        clearPendingUpdates();
         next({
           action: import_service.EchoEvent.DatabaseAction.RESET,
           batch: {
@@ -137,7 +179,7 @@ var DataServiceHost = class {
         const { batch, meta } = message;
         (0, import_invariant2.invariant)(!meta.clientTag, "Unexpected client tag in mutation message", {
           F: __dxlog_file2,
-          L: 69,
+          L: 131,
           S: this,
           A: [
             "!(meta as any).clientTag",
@@ -149,7 +191,7 @@ var DataServiceHost = class {
           meta
         }, {
           F: __dxlog_file2,
-          L: 70,
+          L: 132,
           S: this,
           C: (f, a) => f(...a)
         });
@@ -163,21 +205,37 @@ var DataServiceHost = class {
           });
         });
         if (clientTag) {
+          flushPendingUpdate();
           (0, import_echo_db.tagMutationsInBatch)(batch, clientTag, 0);
+          next({
+            clientTag,
+            feedKey: message.meta.feedKey,
+            seq: message.meta.seq,
+            batch
+          });
+        } else {
+          for (const obj of batch.objects ?? []) {
+            const newCount = (mutationsPerObject.get(obj.objectId) ?? 0) + 1;
+            mutationsPerObject.set(obj.objectId, newCount);
+          }
+          for (const obj of batch.objects ?? []) {
+            if ((mutationsPerObject.get(obj.objectId) ?? 0) < MUTATION_LIMIT_PER_OBJECT) {
+              pendingUpdates.push(obj);
+            }
+          }
+          if (this._deferEvents) {
+            updateScheduler.trigger();
+          } else {
+            flushPendingUpdate();
+          }
         }
-        next({
-          clientTag,
-          feedKey: message.meta.feedKey,
-          seq: message.meta.seq,
-          batch
-        });
       });
     });
   }
   async write(request) {
     (0, import_invariant2.invariant)(!this._ctx.disposed, "Cannot write to closed DataServiceHost", {
       F: __dxlog_file2,
-      L: 96,
+      L: 177,
       S: this,
       A: [
         "!this._ctx.disposed",
@@ -186,7 +244,7 @@ var DataServiceHost = class {
     });
     (0, import_invariant2.invariant)(this._writeStream, "Cannot write mutations in readonly mode", {
       F: __dxlog_file2,
-      L: 97,
+      L: 178,
       S: this,
       A: [
         "this._writeStream",
@@ -198,7 +256,7 @@ var DataServiceHost = class {
       objectCount: request.batch.objects?.length ?? 0
     }, {
       F: __dxlog_file2,
-      L: 99,
+      L: 180,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -212,7 +270,7 @@ var DataServiceHost = class {
             seq: receipt2.seq
           }, {
             F: __dxlog_file2,
-            L: 108,
+            L: 189,
             S: this,
             C: (f, a) => f(...a)
           });
@@ -269,8 +327,8 @@ var DatabaseHost = class {
   createSnapshot() {
     return this._itemDemuxer.createSnapshot();
   }
-  createDataServiceHost() {
-    return new DataServiceHost(this._itemManager, this._itemDemuxer, this._flush, this._outboundStream ?? void 0);
+  createDataServiceHost(opts = {}) {
+    return new DataServiceHost(this._itemManager, this._itemDemuxer, this._flush, this._outboundStream ?? void 0, opts);
   }
 };
 
@@ -463,7 +521,7 @@ var DataServiceImpl = class {
 
 // packages/core/echo/echo-pipeline/src/metadata/metadata-store.ts
 var import_crc_32 = __toESM(require("crc-32"));
-var import_async = require("@dxos/async");
+var import_async2 = require("@dxos/async");
 var import_invariant4 = require("@dxos/invariant");
 var import_keys3 = require("@dxos/keys");
 var import_log3 = require("@dxos/log");
@@ -496,7 +554,7 @@ var MetadataStore = class {
     this._metadata = emptyEchoMetadata();
     this._spaceLargeMetadata = new import_util3.ComplexMap(import_keys3.PublicKey.hash);
     this._metadataFile = void 0;
-    this.update = new import_async.Event();
+    this.update = new import_async2.Event();
   }
   get metadata() {
     return this._metadata;
@@ -755,18 +813,18 @@ var MetadataStore = class {
   }
 };
 _ts_decorate([
-  import_async.synchronized
+  import_async2.synchronized
 ], MetadataStore.prototype, "load", null);
 _ts_decorate([
-  import_async.synchronized
+  import_async2.synchronized
 ], MetadataStore.prototype, "_save", null);
 _ts_decorate([
-  import_async.synchronized
+  import_async2.synchronized
 ], MetadataStore.prototype, "_saveSpaceLargeMetadata", null);
 var fromBytesInt32 = (buf) => buf.readInt32LE(0);
 
 // packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts
-var import_async3 = require("@dxos/async");
+var import_async4 = require("@dxos/async");
 var import_context3 = require("@dxos/context");
 var import_debug3 = require("@dxos/debug");
 var import_feed_store = require("@dxos/feed-store");
@@ -807,7 +865,7 @@ var createMessageSelector = (timeframeClock) => {
 };
 
 // packages/core/echo/echo-pipeline/src/pipeline/timeframe-clock.ts
-var import_async2 = require("@dxos/async");
+var import_async3 = require("@dxos/async");
 var import_debug2 = require("@dxos/debug");
 var import_log5 = require("@dxos/log");
 var import_timeframe = require("@dxos/timeframe");
@@ -837,7 +895,7 @@ var startAfter = (timeframe) => timeframe.frames().map(([feedKey, index]) => ({
 var TimeframeClock = class {
   constructor(_timeframe = new import_timeframe.Timeframe()) {
     this._timeframe = _timeframe;
-    this.update = new import_async2.Event();
+    this.update = new import_async3.Event();
     this._pendingTimeframe = _timeframe;
   }
   /**
@@ -921,7 +979,7 @@ var PipelineState = class {
     this._timeframeClock = _timeframeClock;
     this._ctx = new import_context3.Context();
     this.timeframeUpdate = this._timeframeClock.update;
-    this.stalled = new import_async3.Event();
+    this.stalled = new import_async4.Event();
     this._startTimeframe = new import_timeframe2.Timeframe();
     this._reachedTarget = false;
   }
@@ -995,7 +1053,7 @@ var PipelineState = class {
           done = true;
           this._reachedTarget = true;
         }),
-        (0, import_async3.sleepWithContext)(this._ctx, timeout).then(() => {
+        (0, import_async4.sleepWithContext)(this._ctx, timeout).then(() => {
           if (done) {
             return;
           }
@@ -1024,8 +1082,8 @@ var Pipeline = class {
     // External state accessor.
     this._state = new PipelineState(this._feeds, this._timeframeClock);
     // Waits for the message consumer to process the message and yield control back to the pipeline.
-    this._processingTrigger = new import_async3.Trigger().wake();
-    this._pauseTrigger = new import_async3.Trigger().wake();
+    this._processingTrigger = new import_async4.Trigger().wake();
+    this._pauseTrigger = new import_async4.Trigger().wake();
     // Pending downloads.
     this._downloads = new import_util4.ComplexMap((value) => import_keys4.PublicKey.hash(value.key));
     this._isStopping = false;
@@ -1303,23 +1361,23 @@ var Pipeline = class {
   }
 };
 _ts_decorate3([
-  import_async3.synchronized
+  import_async4.synchronized
 ], Pipeline.prototype, "start", null);
 _ts_decorate3([
-  import_async3.synchronized
+  import_async4.synchronized
 ], Pipeline.prototype, "stop", null);
 _ts_decorate3([
-  import_async3.synchronized
+  import_async4.synchronized
 ], Pipeline.prototype, "setCursor", null);
 _ts_decorate3([
-  import_async3.synchronized
+  import_async4.synchronized
 ], Pipeline.prototype, "pause", null);
 _ts_decorate3([
-  import_async3.synchronized
+  import_async4.synchronized
 ], Pipeline.prototype, "unpause", null);
 
 // packages/core/echo/echo-pipeline/src/space/auth.ts
-var import_async4 = require("@dxos/async");
+var import_async5 = require("@dxos/async");
 var import_context4 = require("@dxos/context");
 var import_crypto2 = require("@dxos/crypto");
 var import_invariant7 = require("@dxos/invariant");
@@ -1377,7 +1435,7 @@ var AuthExtension = class extends import_teleport.RpcExtension {
   }
   async onOpen(context) {
     await super.onOpen(context);
-    (0, import_async4.scheduleTask)(this._ctx, async () => {
+    (0, import_async5.scheduleTask)(this._ctx, async () => {
       try {
         const challenge = (0, import_crypto2.randomBytes)(32);
         const { credential } = await this.rpc.AuthService.authenticate({
@@ -1402,7 +1460,7 @@ var AuthExtension = class extends import_teleport.RpcExtension {
             "'credential not verified'"
           ]
         });
-        (0, import_async4.runInContext)(this._ctx, () => this._authParams.onAuthSuccess());
+        (0, import_async5.runInContext)(this._ctx, () => this._authParams.onAuthSuccess());
       } catch (err) {
         (0, import_log7.log)("auth failed", err, {
           F: __dxlog_file8,
@@ -1426,7 +1484,7 @@ var AuthExtension = class extends import_teleport.RpcExtension {
 };
 
 // packages/core/echo/echo-pipeline/src/space/space.ts
-var import_async7 = require("@dxos/async");
+var import_async8 = require("@dxos/async");
 var import_invariant9 = require("@dxos/invariant");
 var import_log10 = require("@dxos/log");
 var import_credentials4 = require("@dxos/protocols/proto/dxos/halo/credentials");
@@ -1434,7 +1492,7 @@ var import_tracing3 = require("@dxos/tracing");
 var import_util7 = require("@dxos/util");
 
 // packages/core/echo/echo-pipeline/src/space/control-pipeline.ts
-var import_async5 = require("@dxos/async");
+var import_async6 = require("@dxos/async");
 var import_context5 = require("@dxos/context");
 var import_credentials = require("@dxos/credentials");
 var import_keys5 = require("@dxos/keys");
@@ -1464,8 +1522,8 @@ var ControlPipeline = class ControlPipeline2 {
     this.onFeedAdmitted = new import_util5.Callback();
     this._usage = new import_tracing.TimeUsageCounter();
     this._mutations = new import_tracing.TimeSeriesCounter();
-    this._snapshotTask = new import_async5.DeferredTask(this._ctx, async () => {
-      await (0, import_async5.sleepWithContext)(this._ctx, CONTROL_PIPELINE_SNAPSHOT_DELAY);
+    this._snapshotTask = new import_async6.DeferredTask(this._ctx, async () => {
+      await (0, import_async6.sleepWithContext)(this._ctx, CONTROL_PIPELINE_SNAPSHOT_DELAY);
       await this._saveSnapshot();
     });
     this._spaceKey = spaceKey;
@@ -1688,11 +1746,11 @@ _ts_decorate4([
 ], ControlPipeline.prototype, "_processMessage", null);
 ControlPipeline = _ts_decorate4([
   import_tracing.trace.resource(),
-  (0, import_async5.trackLeaks)("start", "stop")
+  (0, import_async6.trackLeaks)("start", "stop")
 ], ControlPipeline);
 
 // packages/core/echo/echo-pipeline/src/space/data-pipeline.ts
-var import_async6 = require("@dxos/async");
+var import_async7 = require("@dxos/async");
 var import_context6 = require("@dxos/context");
 var import_credentials3 = require("@dxos/credentials");
 var import_echo_db3 = require("@dxos/echo-db");
@@ -1731,7 +1789,7 @@ var DataPipeline = class DataPipeline2 {
     this._mutations = new import_tracing2.TimeSeriesCounter();
     this.currentEpoch = void 0;
     this.appliedEpoch = void 0;
-    this.onNewEpoch = new import_async6.Event();
+    this.onNewEpoch = new import_async7.Event();
   }
   get isOpen() {
     return this._isOpen;
@@ -1794,7 +1852,7 @@ var DataPipeline = class DataPipeline2 {
     this.databaseHost = new DatabaseHost(feedWriter, () => this._flush());
     this.itemManager = new import_echo_db3.ItemManager(this._params.modelFactory);
     await this.databaseHost.open(this.itemManager, this._params.modelFactory);
-    (0, import_async6.scheduleTask)(this._ctx, async () => {
+    (0, import_async7.scheduleTask)(this._ctx, async () => {
       await this._consumePipeline();
     });
     this._isOpen = true;
@@ -1913,7 +1971,7 @@ var DataPipeline = class DataPipeline2 {
       span.end();
       if (++messageCounter > 1e3) {
         messageCounter = 0;
-        await (0, import_async6.sleep)(1);
+        await (0, import_async7.sleep)(1);
       }
     }
   }
@@ -1993,7 +2051,7 @@ var DataPipeline = class DataPipeline2 {
       }
     });
     this._epochCtx = ctx;
-    (0, import_async6.scheduleTask)(ctx, async () => {
+    (0, import_async7.scheduleTask)(ctx, async () => {
       if (!this._isOpen) {
         return;
       }
@@ -2114,19 +2172,19 @@ _ts_decorate5([
   import_tracing2.trace.metricsCounter()
 ], DataPipeline.prototype, "_mutations", void 0);
 _ts_decorate5([
-  import_async6.synchronized
+  import_async7.synchronized
 ], DataPipeline.prototype, "open", null);
 _ts_decorate5([
-  import_async6.synchronized
+  import_async7.synchronized
 ], DataPipeline.prototype, "close", null);
 _ts_decorate5([
-  import_async6.synchronized
+  import_async7.synchronized
 ], DataPipeline.prototype, "_processEpoch", null);
 _ts_decorate5([
-  import_async6.synchronized
+  import_async7.synchronized
 ], DataPipeline.prototype, "createEpoch", null);
 DataPipeline = _ts_decorate5([
-  (0, import_async6.trackLeaks)("open", "close"),
+  (0, import_async7.trackLeaks)("open", "close"),
   import_tracing2.trace.resource()
 ], DataPipeline);
 
@@ -2144,9 +2202,9 @@ function _ts_decorate6(decorators, target, key, desc) {
 var __dxlog_file11 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space.ts";
 var Space = class Space2 {
   constructor(params) {
-    this._addFeedLock = new import_async7.Lock();
+    this._addFeedLock = new import_async8.Lock();
     this.onCredentialProcessed = new import_util7.Callback();
-    this.stateUpdate = new import_async7.Event();
+    this.stateUpdate = new import_async8.Event();
     this._isOpen = false;
     (0, import_invariant9.invariant)(params.spaceKey && params.feedProvider, void 0, {
       F: __dxlog_file11,
@@ -2365,22 +2423,22 @@ _ts_decorate6([
   import_tracing3.trace.info()
 ], Space.prototype, "key", null);
 _ts_decorate6([
-  import_async7.synchronized,
+  import_async8.synchronized,
   import_tracing3.trace.span()
 ], Space.prototype, "open", null);
 _ts_decorate6([
-  import_async7.synchronized
+  import_async8.synchronized
 ], Space.prototype, "close", null);
 _ts_decorate6([
-  import_async7.synchronized
+  import_async8.synchronized
 ], Space.prototype, "initializeDataPipeline", null);
 Space = _ts_decorate6([
-  (0, import_async7.trackLeaks)("open", "close"),
+  (0, import_async8.trackLeaks)("open", "close"),
   import_tracing3.trace.resource()
 ], Space);
 
 // packages/core/echo/echo-pipeline/src/space/space-manager.ts
-var import_async8 = require("@dxos/async");
+var import_async9 = require("@dxos/async");
 var import_debug4 = require("@dxos/debug");
 var import_keys7 = require("@dxos/keys");
 var import_log12 = require("@dxos/log");
@@ -2678,13 +2736,13 @@ var SpaceManager = class SpaceManager2 {
   }
 };
 _ts_decorate8([
-  import_async8.synchronized
+  import_async9.synchronized
 ], SpaceManager.prototype, "open", null);
 _ts_decorate8([
-  import_async8.synchronized
+  import_async9.synchronized
 ], SpaceManager.prototype, "close", null);
 SpaceManager = _ts_decorate8([
-  (0, import_async8.trackLeaks)("open", "close")
+  (0, import_async9.trackLeaks)("open", "close")
 ], SpaceManager);
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {