@dxos/echo-pipeline 0.3.3-main.f2ca85a → 0.3.3-main.f990aad

package/dist/lib/node/testing/index.cjs

@@ -103,6 +103,7 @@ var TestFeedBuilder = class extends import_testing.TestBuilder {
 };
 
 // packages/core/echo/echo-pipeline/src/db-host/data-service-host.ts
+var import_async = require("@dxos/async");
 var import_codec_protobuf = require("@dxos/codec-protobuf");
 var import_context = require("@dxos/context");
 var import_echo_db = require("@dxos/echo-db");
@@ -111,12 +112,14 @@ var import_log = require("@dxos/log");
 var import_service = require("@dxos/protocols/proto/dxos/echo/service");
 var import_util = require("@dxos/util");
 var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/data-service-host.ts";
+var MUTATION_LIMIT_PER_OBJECT = 10;
 var DataServiceHost = class {
-  constructor(_itemManager, _itemDemuxer, _flush, _writeStream) {
+  constructor(_itemManager, _itemDemuxer, _flush, _writeStream, _opts = {}) {
     this._itemManager = _itemManager;
     this._itemDemuxer = _itemDemuxer;
     this._flush = _flush;
     this._writeStream = _writeStream;
+    this._opts = _opts;
     this._ctx = new import_context.Context();
     this._clientTagMap = new import_util.ComplexMap(([feedKey, seq]) => `${feedKey.toHex()}:${seq}`);
   }
@@ -125,6 +128,9 @@ var DataServiceHost = class {
   async close() {
     await this._ctx.dispose();
   }
+  get _deferEvents() {
+    return this._opts.deferEvents ?? true;
+  }
   /**
   * Real-time subscription to data objects in a space.
   */
@@ -137,7 +143,43 @@ var DataServiceHost = class {
           objects
         }
       });
+      const updateScheduler = new import_async.UpdateScheduler(ctx, async () => {
+        flushPendingUpdate();
+      }, {
+        maxFrequency: 10
+      });
+      const pendingUpdates = [];
+      const mutationsPerObject = /* @__PURE__ */ new Map();
+      const clearPendingUpdates = () => {
+        pendingUpdates.length = 0;
+        mutationsPerObject.clear();
+      };
+      const flushPendingUpdate = () => {
+        const stagedEvents = [];
+        const objectsWithSnapshots = /* @__PURE__ */ new Set();
+        for (const [id, count] of mutationsPerObject) {
+          if (count >= MUTATION_LIMIT_PER_OBJECT) {
+            objectsWithSnapshots.add(id);
+            const entity = this._itemManager.entities.get(id);
+            if (entity) {
+              stagedEvents.push(entity.createSnapshot());
+            }
+          }
+        }
+        for (const obj of pendingUpdates) {
+          if (!objectsWithSnapshots.has(obj.objectId)) {
+            stagedEvents.push(obj);
+          }
+        }
+        next({
+          batch: {
+            objects: stagedEvents
+          }
+        });
+        clearPendingUpdates();
+      };
       this._itemDemuxer.snapshot.on(ctx, (snapshot) => {
+        clearPendingUpdates();
         next({
           action: import_service.EchoEvent.DatabaseAction.RESET,
           batch: {
@@ -149,7 +191,7 @@ var DataServiceHost = class {
         const { batch, meta } = message;
         (0, import_invariant2.invariant)(!meta.clientTag, "Unexpected client tag in mutation message", {
           F: __dxlog_file2,
-          L: 69,
+          L: 131,
           S: this,
           A: [
             "!(meta as any).clientTag",
@@ -161,7 +203,7 @@ var DataServiceHost = class {
           meta
         }, {
           F: __dxlog_file2,
-          L: 70,
+          L: 132,
           S: this,
           C: (f, a) => f(...a)
         });
@@ -175,21 +217,37 @@ var DataServiceHost = class {
           });
         });
         if (clientTag) {
+          flushPendingUpdate();
           (0, import_echo_db.tagMutationsInBatch)(batch, clientTag, 0);
+          next({
+            clientTag,
+            feedKey: message.meta.feedKey,
+            seq: message.meta.seq,
+            batch
+          });
+        } else {
+          for (const obj of batch.objects ?? []) {
+            const newCount = (mutationsPerObject.get(obj.objectId) ?? 0) + 1;
+            mutationsPerObject.set(obj.objectId, newCount);
+          }
+          for (const obj of batch.objects ?? []) {
+            if ((mutationsPerObject.get(obj.objectId) ?? 0) < MUTATION_LIMIT_PER_OBJECT) {
+              pendingUpdates.push(obj);
+            }
+          }
+          if (this._deferEvents) {
+            updateScheduler.trigger();
+          } else {
+            flushPendingUpdate();
+          }
         }
-        next({
-          clientTag,
-          feedKey: message.meta.feedKey,
-          seq: message.meta.seq,
-          batch
-        });
       });
     });
   }
   async write(request) {
     (0, import_invariant2.invariant)(!this._ctx.disposed, "Cannot write to closed DataServiceHost", {
       F: __dxlog_file2,
-      L: 96,
+      L: 177,
       S: this,
       A: [
         "!this._ctx.disposed",
@@ -198,7 +256,7 @@ var DataServiceHost = class {
     });
     (0, import_invariant2.invariant)(this._writeStream, "Cannot write mutations in readonly mode", {
       F: __dxlog_file2,
-      L: 97,
+      L: 178,
       S: this,
       A: [
         "this._writeStream",
@@ -210,7 +268,7 @@ var DataServiceHost = class {
       objectCount: request.batch.objects?.length ?? 0
     }, {
       F: __dxlog_file2,
-      L: 99,
+      L: 180,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -224,7 +282,7 @@ var DataServiceHost = class {
             seq: receipt2.seq
           }, {
             F: __dxlog_file2,
-            L: 108,
+            L: 189,
             S: this,
             C: (f, a) => f(...a)
           });
@@ -281,8 +339,8 @@ var DatabaseHost = class {
   createSnapshot() {
     return this._itemDemuxer.createSnapshot();
   }
-  createDataServiceHost() {
-    return new DataServiceHost(this._itemManager, this._itemDemuxer, this._flush, this._outboundStream ?? void 0);
+  createDataServiceHost(opts = {}) {
+    return new DataServiceHost(this._itemManager, this._itemDemuxer, this._flush, this._outboundStream ?? void 0, opts);
   }
 };
 
@@ -475,7 +533,7 @@ var DataServiceImpl = class {
 
 // packages/core/echo/echo-pipeline/src/metadata/metadata-store.ts
 var import_crc_32 = __toESM(require("crc-32"));
-var import_async = require("@dxos/async");
+var import_async2 = require("@dxos/async");
 var import_invariant4 = require("@dxos/invariant");
 var import_keys3 = require("@dxos/keys");
 var import_log3 = require("@dxos/log");
@@ -508,7 +566,7 @@ var MetadataStore = class {
     this._metadata = emptyEchoMetadata();
     this._spaceLargeMetadata = new import_util3.ComplexMap(import_keys3.PublicKey.hash);
     this._metadataFile = void 0;
-    this.update = new import_async.Event();
+    this.update = new import_async2.Event();
   }
   get metadata() {
     return this._metadata;
@@ -767,18 +825,18 @@ var MetadataStore = class {
   }
 };
 _ts_decorate([
-  import_async.synchronized
+  import_async2.synchronized
 ], MetadataStore.prototype, "load", null);
 _ts_decorate([
-  import_async.synchronized
+  import_async2.synchronized
 ], MetadataStore.prototype, "_save", null);
 _ts_decorate([
-  import_async.synchronized
+  import_async2.synchronized
 ], MetadataStore.prototype, "_saveSpaceLargeMetadata", null);
 var fromBytesInt32 = (buf) => buf.readInt32LE(0);
 
 // packages/core/echo/echo-pipeline/src/space/auth.ts
-var import_async2 = require("@dxos/async");
+var import_async3 = require("@dxos/async");
 var import_context3 = require("@dxos/context");
 var import_crypto2 = require("@dxos/crypto");
 var import_invariant5 = require("@dxos/invariant");
@@ -836,7 +894,7 @@ var AuthExtension = class extends import_teleport.RpcExtension {
   }
   async onOpen(context) {
     await super.onOpen(context);
-    (0, import_async2.scheduleTask)(this._ctx, async () => {
+    (0, import_async3.scheduleTask)(this._ctx, async () => {
       try {
         const challenge = (0, import_crypto2.randomBytes)(32);
         const { credential } = await this.rpc.AuthService.authenticate({
@@ -861,7 +919,7 @@ var AuthExtension = class extends import_teleport.RpcExtension {
             "'credential not verified'"
           ]
         });
-        (0, import_async2.runInContext)(this._ctx, () => this._authParams.onAuthSuccess());
+        (0, import_async3.runInContext)(this._ctx, () => this._authParams.onAuthSuccess());
       } catch (err) {
         (0, import_log4.log)("auth failed", err, {
           F: __dxlog_file5,
@@ -885,7 +943,7 @@ var AuthExtension = class extends import_teleport.RpcExtension {
 };
 
 // packages/core/echo/echo-pipeline/src/space/space.ts
-var import_async7 = require("@dxos/async");
+var import_async8 = require("@dxos/async");
 var import_invariant9 = require("@dxos/invariant");
 var import_log10 = require("@dxos/log");
 var import_credentials4 = require("@dxos/protocols/proto/dxos/halo/credentials");
@@ -893,7 +951,7 @@ var import_tracing3 = require("@dxos/tracing");
 var import_util7 = require("@dxos/util");
 
 // packages/core/echo/echo-pipeline/src/space/control-pipeline.ts
-var import_async5 = require("@dxos/async");
+var import_async6 = require("@dxos/async");
 var import_context5 = require("@dxos/context");
 var import_credentials = require("@dxos/credentials");
 var import_keys5 = require("@dxos/keys");
@@ -904,7 +962,7 @@ var import_tracing = require("@dxos/tracing");
 var import_util5 = require("@dxos/util");
 
 // packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts
-var import_async4 = require("@dxos/async");
+var import_async5 = require("@dxos/async");
 var import_context4 = require("@dxos/context");
 var import_debug3 = require("@dxos/debug");
 var import_feed_store = require("@dxos/feed-store");
@@ -945,7 +1003,7 @@ var createMessageSelector = (timeframeClock) => {
 };
 
 // packages/core/echo/echo-pipeline/src/pipeline/timeframe-clock.ts
-var import_async3 = require("@dxos/async");
+var import_async4 = require("@dxos/async");
 var import_debug2 = require("@dxos/debug");
 var import_log6 = require("@dxos/log");
 var import_timeframe = require("@dxos/timeframe");
@@ -971,7 +1029,7 @@ var startAfter = (timeframe) => timeframe.frames().map(([feedKey, index]) => ({
 var TimeframeClock = class {
   constructor(_timeframe = new import_timeframe.Timeframe()) {
     this._timeframe = _timeframe;
-    this.update = new import_async3.Event();
+    this.update = new import_async4.Event();
     this._pendingTimeframe = _timeframe;
   }
   /**
@@ -1055,7 +1113,7 @@ var PipelineState = class {
     this._timeframeClock = _timeframeClock;
     this._ctx = new import_context4.Context();
     this.timeframeUpdate = this._timeframeClock.update;
-    this.stalled = new import_async4.Event();
+    this.stalled = new import_async5.Event();
     this._startTimeframe = new import_timeframe2.Timeframe();
     this._reachedTarget = false;
   }
@@ -1129,7 +1187,7 @@ var PipelineState = class {
           done = true;
           this._reachedTarget = true;
         }),
-        (0, import_async4.sleepWithContext)(this._ctx, timeout).then(() => {
+        (0, import_async5.sleepWithContext)(this._ctx, timeout).then(() => {
           if (done) {
             return;
           }
@@ -1158,8 +1216,8 @@ var Pipeline = class {
     // External state accessor.
     this._state = new PipelineState(this._feeds, this._timeframeClock);
     // Waits for the message consumer to process the message and yield control back to the pipeline.
-    this._processingTrigger = new import_async4.Trigger().wake();
-    this._pauseTrigger = new import_async4.Trigger().wake();
+    this._processingTrigger = new import_async5.Trigger().wake();
+    this._pauseTrigger = new import_async5.Trigger().wake();
     // Pending downloads.
     this._downloads = new import_util4.ComplexMap((value) => import_keys4.PublicKey.hash(value.key));
     this._isStopping = false;
@@ -1437,19 +1495,19 @@ var Pipeline = class {
   }
 };
 _ts_decorate3([
-  import_async4.synchronized
+  import_async5.synchronized
 ], Pipeline.prototype, "start", null);
 _ts_decorate3([
-  import_async4.synchronized
+  import_async5.synchronized
 ], Pipeline.prototype, "stop", null);
 _ts_decorate3([
-  import_async4.synchronized
+  import_async5.synchronized
 ], Pipeline.prototype, "setCursor", null);
 _ts_decorate3([
-  import_async4.synchronized
+  import_async5.synchronized
 ], Pipeline.prototype, "pause", null);
 _ts_decorate3([
-  import_async4.synchronized
+  import_async5.synchronized
 ], Pipeline.prototype, "unpause", null);
 
 // packages/core/echo/echo-pipeline/src/space/control-pipeline.ts
@@ -1474,8 +1532,8 @@ var ControlPipeline = class ControlPipeline2 {
     this.onFeedAdmitted = new import_util5.Callback();
     this._usage = new import_tracing.TimeUsageCounter();
     this._mutations = new import_tracing.TimeSeriesCounter();
-    this._snapshotTask = new import_async5.DeferredTask(this._ctx, async () => {
-      await (0, import_async5.sleepWithContext)(this._ctx, CONTROL_PIPELINE_SNAPSHOT_DELAY);
+    this._snapshotTask = new import_async6.DeferredTask(this._ctx, async () => {
+      await (0, import_async6.sleepWithContext)(this._ctx, CONTROL_PIPELINE_SNAPSHOT_DELAY);
       await this._saveSnapshot();
     });
     this._spaceKey = spaceKey;
@@ -1698,11 +1756,11 @@ _ts_decorate4([
 ], ControlPipeline.prototype, "_processMessage", null);
 ControlPipeline = _ts_decorate4([
   import_tracing.trace.resource(),
-  (0, import_async5.trackLeaks)("start", "stop")
+  (0, import_async6.trackLeaks)("start", "stop")
 ], ControlPipeline);
 
 // packages/core/echo/echo-pipeline/src/space/data-pipeline.ts
-var import_async6 = require("@dxos/async");
+var import_async7 = require("@dxos/async");
 var import_context6 = require("@dxos/context");
 var import_credentials3 = require("@dxos/credentials");
 var import_echo_db3 = require("@dxos/echo-db");
@@ -1741,7 +1799,7 @@ var DataPipeline = class DataPipeline2 {
     this._mutations = new import_tracing2.TimeSeriesCounter();
     this.currentEpoch = void 0;
     this.appliedEpoch = void 0;
-    this.onNewEpoch = new import_async6.Event();
+    this.onNewEpoch = new import_async7.Event();
   }
   get isOpen() {
     return this._isOpen;
@@ -1804,7 +1862,7 @@ var DataPipeline = class DataPipeline2 {
     this.databaseHost = new DatabaseHost(feedWriter, () => this._flush());
     this.itemManager = new import_echo_db3.ItemManager(this._params.modelFactory);
     await this.databaseHost.open(this.itemManager, this._params.modelFactory);
-    (0, import_async6.scheduleTask)(this._ctx, async () => {
+    (0, import_async7.scheduleTask)(this._ctx, async () => {
       await this._consumePipeline();
     });
     this._isOpen = true;
@@ -1923,7 +1981,7 @@ var DataPipeline = class DataPipeline2 {
       span.end();
       if (++messageCounter > 1e3) {
         messageCounter = 0;
-        await (0, import_async6.sleep)(1);
+        await (0, import_async7.sleep)(1);
       }
     }
   }
@@ -2003,7 +2061,7 @@ var DataPipeline = class DataPipeline2 {
       }
     });
     this._epochCtx = ctx;
-    (0, import_async6.scheduleTask)(ctx, async () => {
+    (0, import_async7.scheduleTask)(ctx, async () => {
       if (!this._isOpen) {
         return;
       }
@@ -2124,19 +2182,19 @@ _ts_decorate5([
   import_tracing2.trace.metricsCounter()
 ], DataPipeline.prototype, "_mutations", void 0);
 _ts_decorate5([
-  import_async6.synchronized
+  import_async7.synchronized
 ], DataPipeline.prototype, "open", null);
 _ts_decorate5([
-  import_async6.synchronized
+  import_async7.synchronized
 ], DataPipeline.prototype, "close", null);
 _ts_decorate5([
-  import_async6.synchronized
+  import_async7.synchronized
 ], DataPipeline.prototype, "_processEpoch", null);
 _ts_decorate5([
-  import_async6.synchronized
+  import_async7.synchronized
 ], DataPipeline.prototype, "createEpoch", null);
 DataPipeline = _ts_decorate5([
-  (0, import_async6.trackLeaks)("open", "close"),
+  (0, import_async7.trackLeaks)("open", "close"),
   import_tracing2.trace.resource()
 ], DataPipeline);
 
@@ -2154,9 +2212,9 @@ function _ts_decorate6(decorators, target, key, desc) {
 var __dxlog_file11 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space.ts";
 var Space = class Space2 {
   constructor(params) {
-    this._addFeedLock = new import_async7.Lock();
+    this._addFeedLock = new import_async8.Lock();
     this.onCredentialProcessed = new import_util7.Callback();
-    this.stateUpdate = new import_async7.Event();
+    this.stateUpdate = new import_async8.Event();
     this._isOpen = false;
     (0, import_invariant9.invariant)(params.spaceKey && params.feedProvider, void 0, {
       F: __dxlog_file11,
@@ -2375,22 +2433,22 @@ _ts_decorate6([
   import_tracing3.trace.info()
 ], Space.prototype, "key", null);
 _ts_decorate6([
-  import_async7.synchronized,
+  import_async8.synchronized,
   import_tracing3.trace.span()
 ], Space.prototype, "open", null);
 _ts_decorate6([
-  import_async7.synchronized
+  import_async8.synchronized
 ], Space.prototype, "close", null);
 _ts_decorate6([
-  import_async7.synchronized
+  import_async8.synchronized
 ], Space.prototype, "initializeDataPipeline", null);
 Space = _ts_decorate6([
-  (0, import_async7.trackLeaks)("open", "close"),
+  (0, import_async8.trackLeaks)("open", "close"),
   import_tracing3.trace.resource()
 ], Space);
 
 // packages/core/echo/echo-pipeline/src/space/space-manager.ts
-var import_async8 = require("@dxos/async");
+var import_async9 = require("@dxos/async");
 var import_debug4 = require("@dxos/debug");
 var import_keys7 = require("@dxos/keys");
 var import_log12 = require("@dxos/log");
@@ -2688,13 +2746,13 @@ var SpaceManager = class SpaceManager2 {
   }
 };
 _ts_decorate8([
-  import_async8.synchronized
+  import_async9.synchronized
 ], SpaceManager.prototype, "open", null);
 _ts_decorate8([
-  import_async8.synchronized
+  import_async9.synchronized
 ], SpaceManager.prototype, "close", null);
 SpaceManager = _ts_decorate8([
-  (0, import_async8.trackLeaks)("open", "close")
+  (0, import_async9.trackLeaks)("open", "close")
 ], SpaceManager);
 
 // packages/core/echo/echo-pipeline/src/testing/test-agent-builder.ts
@@ -2871,7 +2929,7 @@ var TestAgent = class {
 };
 
 // packages/core/echo/echo-pipeline/src/testing/util.ts
-var import_async9 = require("@dxos/async");
+var import_async10 = require("@dxos/async");
 var import_document_model2 = require("@dxos/document-model");
 var import_echo_db4 = require("@dxos/echo-db");
 var import_testing2 = require("@dxos/feed-store/testing");
@@ -2933,11 +2991,11 @@ var testLocalDatabase = async (create, check = create) => {
       ]
     }
   });
-  await (0, import_async9.asyncTimeout)(check.databaseHost._itemDemuxer.mutation.waitForCondition(() => check.itemManager.entities.has(objectId)), 2e3);
+  await (0, import_async10.asyncTimeout)(check.databaseHost._itemDemuxer.mutation.waitForCondition(() => check.itemManager.entities.has(objectId)), 2e3);
 };
 
 // packages/core/echo/echo-pipeline/src/testing/database-test-rig.ts
-var import_async10 = require("@dxos/async");
+var import_async11 = require("@dxos/async");
 var import_document_model3 = require("@dxos/document-model");
 var import_echo_db5 = require("@dxos/echo-db");
 var import_invariant10 = require("@dxos/invariant");
@@ -2970,7 +3028,7 @@ var DatabaseTestPeer = class {
     this.snapshots = /* @__PURE__ */ new Map();
     this.confirmed = -1;
     this.timeframe = new import_timeframe6.Timeframe();
-    this._onConfirm = new import_async10.Event();
+    this._onConfirm = new import_async11.Event();
     this._writes = /* @__PURE__ */ new Set();
   }
   async open() {
@@ -2991,7 +3049,7 @@ var DatabaseTestPeer = class {
           options: {
             afterWrite
           },
-          trigger: new import_async10.Trigger()
+          trigger: new import_async11.Trigger()
         };
         this._writes.add(request);
         await request.trigger.wait();
@@ -3005,7 +3063,9 @@ var DatabaseTestPeer = class {
     }
     this.items = new import_echo_db5.ItemManager(this.modelFactory);
     this.proxy = new import_echo_db5.DatabaseProxy({
-      service: this.host.createDataServiceHost(),
+      service: this.host.createDataServiceHost({
+        deferEvents: false
+      }),
       itemManager: this.items,
       spaceKey: this.spaceKey
     });