@dxos/echo-pipeline 0.3.3-main.e0ded2e → 0.3.3-main.ef9deca

package/dist/types/src/db-host/data-service-host.d.ts

@@ -3,6 +3,12 @@ import { type ItemDemuxer, type ItemManager } from '@dxos/echo-db';
 import { type FeedWriter } from '@dxos/feed-store';
 import { type DataMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { EchoEvent, type MutationReceipt, type WriteRequest } from '@dxos/protocols/proto/dxos/echo/service';
+export type DataServiceHostOptions = {
+    /**
+     * @default true
+     */
+    deferEvents?: boolean;
+};
 /**
  * Provides methods for DataService for a single space.
  * A DataServiceRouter must be placed before it to route requests to different DataServiceHost instances based on space id.
@@ -11,12 +17,14 @@ export declare class DataServiceHost {
     private readonly _itemManager;
     private readonly _itemDemuxer;
     private readonly _flush;
-    private readonly _writeStream?;
+    private readonly _writeStream;
+    private readonly _opts;
     private readonly _ctx;
     private readonly _clientTagMap;
-    constructor(_itemManager: ItemManager, _itemDemuxer: ItemDemuxer, _flush: () => Promise<void>, _writeStream?: FeedWriter<DataMessage> | undefined);
+    constructor(_itemManager: ItemManager, _itemDemuxer: ItemDemuxer, _flush: () => Promise<void>, _writeStream: FeedWriter<DataMessage> | undefined, _opts?: DataServiceHostOptions);
     open(): Promise<void>;
     close(): Promise<void>;
+    private get _deferEvents();
     /**
      * Real-time subscription to data objects in a space.
      */

package/dist/types/src/db-host/data-service-host.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"data-service-host.d.ts","sourceRoot":"","sources":["../../../../src/db-host/data-service-host.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAE9C,OAAO,EAAuB,KAAK,WAAW,EAAE,KAAK,WAAW,EAAuB,MAAM,eAAe,CAAC;AAC7G,OAAO,EAAE,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAInD,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,sCAAsC,CAAC;AAExE,OAAO,EAAE,SAAS,EAAE,KAAK,eAAe,EAAE,KAAK,YAAY,EAAE,MAAM,yCAAyC,CAAC;AAG7G;;;GAGG;AAEH,qBAAa,eAAe;IAQxB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;IAVhC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiB;IAEtC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAE5B;gBAGiB,YAAY,EAAE,WAAW,EACzB,YAAY,EAAE,WAAW,EACzB,MAAM,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,EAC3B,YAAY,CAAC,qCAAyB;IAGnD,IAAI;IAEJ,KAAK;IAIX;;OAEG;IACH,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;IAkDxB,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC;IAsBtD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B"}
+{"version":3,"file":"data-service-host.d.ts","sourceRoot":"","sources":["../../../../src/db-host/data-service-host.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAE9C,OAAO,EAAuB,KAAK,WAAW,EAAE,KAAK,WAAW,EAAuB,MAAM,eAAe,CAAC;AAC7G,OAAO,EAAE,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAInD,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,sCAAsC,CAAC;AAExE,OAAO,EAAE,SAAS,EAAE,KAAK,eAAe,EAAE,KAAK,YAAY,EAAE,MAAM,yCAAyC,CAAC;AAM7G,MAAM,MAAM,sBAAsB,GAAG;IACnC;;OAEG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB,CAAC;AAEF;;;GAGG;AAEH,qBAAa,eAAe;IAQxB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,KAAK;IAXxB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiB;IAEtC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAE5B;gBAGiB,YAAY,EAAE,WAAW,EACzB,YAAY,EAAE,WAAW,EACzB,MAAM,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,EAC3B,YAAY,EAAE,UAAU,CAAC,WAAW,CAAC,GAAG,SAAS,EACjD,KAAK,GAAE,sBAA2B;IAG/C,IAAI;IAEJ,KAAK;IAIX,OAAO,KAAK,YAAY,GAEvB;IAED;;OAEG;IACH,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;IAmHxB,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC;IAsBtD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B"}

package/dist/types/src/db-host/database-host.d.ts

@@ -3,7 +3,7 @@ import { type FeedWriter } from '@dxos/feed-store';
 import { type ModelFactory } from '@dxos/model-factory';
 import { type DataMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { type EchoSnapshot } from '@dxos/protocols/proto/dxos/echo/snapshot';
-import { DataServiceHost } from './data-service-host';
+import { DataServiceHost, type DataServiceHostOptions } from './data-service-host';
 /**
  * Database backend that operates on two streams: read and write.
  * Mutations are read from the incoming streams and applied to the ItemManager via ItemDemuxer.
@@ -22,6 +22,6 @@ export declare class DatabaseHost {
     close(): Promise<void>;
     getWriteStream(): FeedWriter<DataMessage> | undefined;
     createSnapshot(): EchoSnapshot;
-    createDataServiceHost(): DataServiceHost;
+    createDataServiceHost(opts?: DataServiceHostOptions): DataServiceHost;
 }
 //# sourceMappingURL=database-host.d.ts.map

package/dist/types/src/db-host/database-host.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"database-host.d.ts","sourceRoot":"","sources":["../../../../src/db-host/database-host.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,aAAa,EAAE,WAAW,EAAE,KAAK,WAAW,EAAE,MAAM,eAAe,CAAC;AAClF,OAAO,EAAE,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,0CAA0C,CAAC;AAE7E,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAEtD;;;;GAIG;AACH,qBAAa,YAAY;IAMrB,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,MAAM;IANzB,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,YAAY,CAAe;IAC5B,YAAY,EAAG,WAAW,CAAC;gBAGf,eAAe,EAAE,UAAU,CAAC,WAAW,CAAC,GAAG,SAAS,EACpD,MAAM,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC;IAG9C,IAAI,UAAU,IAAI,OAAO,CAExB;IAED,IAAI,aAAa,kBAEhB;IAEK,IAAI,CAAC,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY;IAQzD,KAAK;IAEX,cAAc,IAAI,UAAU,CAAC,WAAW,CAAC,GAAG,SAAS;IAIrD,cAAc,IAAI,YAAY;IAI9B,qBAAqB;CAGtB"}
+{"version":3,"file":"database-host.d.ts","sourceRoot":"","sources":["../../../../src/db-host/database-host.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,aAAa,EAAE,WAAW,EAAE,KAAK,WAAW,EAAE,MAAM,eAAe,CAAC;AAClF,OAAO,EAAE,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,0CAA0C,CAAC;AAE7E,OAAO,EAAE,eAAe,EAAE,KAAK,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAEnF;;;;GAIG;AACH,qBAAa,YAAY;IAMrB,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,MAAM;IANzB,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,YAAY,CAAe;IAC5B,YAAY,EAAG,WAAW,CAAC;gBAGf,eAAe,EAAE,UAAU,CAAC,WAAW,CAAC,GAAG,SAAS,EACpD,MAAM,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC;IAG9C,IAAI,UAAU,IAAI,OAAO,CAExB;IAED,IAAI,aAAa,kBAEhB;IAEK,IAAI,CAAC,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY;IAQzD,KAAK;IAEX,cAAc,IAAI,UAAU,CAAC,WAAW,CAAC,GAAG,SAAS;IAIrD,cAAc,IAAI,YAAY;IAI9B,qBAAqB,CAAC,IAAI,GAAE,sBAA2B;CASxD"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dxos/echo-pipeline",
-  "version": "0.3.3-main.e0ded2e",
+  "version": "0.3.3-main.ef9deca",
   "description": "ECHO database.",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
@@ -20,36 +20,36 @@
   ],
   "dependencies": {
     "crc-32": "^1.2.2",
-    "@dxos/async": "0.3.3-main.e0ded2e",
-    "@dxos/codec-protobuf": "0.3.3-main.e0ded2e",
-    "@dxos/context": "0.3.3-main.e0ded2e",
-    "@dxos/credentials": "0.3.3-main.e0ded2e",
-    "@dxos/crypto": "0.3.3-main.e0ded2e",
-    "@dxos/debug": "0.3.3-main.e0ded2e",
-    "@dxos/document-model": "0.3.3-main.e0ded2e",
-    "@dxos/echo-db": "0.3.3-main.e0ded2e",
-    "@dxos/feed-store": "0.3.3-main.e0ded2e",
-    "@dxos/hypercore": "0.3.3-main.e0ded2e",
-    "@dxos/invariant": "0.3.3-main.e0ded2e",
-    "@dxos/keyring": "0.3.3-main.e0ded2e",
-    "@dxos/keys": "0.3.3-main.e0ded2e",
-    "@dxos/log": "0.3.3-main.e0ded2e",
-    "@dxos/messaging": "0.3.3-main.e0ded2e",
-    "@dxos/model-factory": "0.3.3-main.e0ded2e",
-    "@dxos/network-manager": "0.3.3-main.e0ded2e",
-    "@dxos/node-std": "0.3.3-main.e0ded2e",
-    "@dxos/protocols": "0.3.3-main.e0ded2e",
-    "@dxos/random-access-storage": "0.3.3-main.e0ded2e",
-    "@dxos/rpc": "0.3.3-main.e0ded2e",
-    "@dxos/teleport": "0.3.3-main.e0ded2e",
-    "@dxos/teleport-extension-gossip": "0.3.3-main.e0ded2e",
-    "@dxos/teleport-extension-object-sync": "0.3.3-main.e0ded2e",
-    "@dxos/teleport-extension-replicator": "0.3.3-main.e0ded2e",
-    "@dxos/text-model": "0.3.3-main.e0ded2e",
-    "@dxos/timeframe": "0.3.3-main.e0ded2e",
-    "@dxos/tracing": "0.3.3-main.e0ded2e",
-    "@dxos/typings": "0.3.3-main.e0ded2e",
-    "@dxos/util": "0.3.3-main.e0ded2e"
+    "@dxos/async": "0.3.3-main.ef9deca",
+    "@dxos/codec-protobuf": "0.3.3-main.ef9deca",
+    "@dxos/context": "0.3.3-main.ef9deca",
+    "@dxos/credentials": "0.3.3-main.ef9deca",
+    "@dxos/crypto": "0.3.3-main.ef9deca",
+    "@dxos/debug": "0.3.3-main.ef9deca",
+    "@dxos/document-model": "0.3.3-main.ef9deca",
+    "@dxos/echo-db": "0.3.3-main.ef9deca",
+    "@dxos/feed-store": "0.3.3-main.ef9deca",
+    "@dxos/hypercore": "0.3.3-main.ef9deca",
+    "@dxos/invariant": "0.3.3-main.ef9deca",
+    "@dxos/keyring": "0.3.3-main.ef9deca",
+    "@dxos/keys": "0.3.3-main.ef9deca",
+    "@dxos/log": "0.3.3-main.ef9deca",
+    "@dxos/messaging": "0.3.3-main.ef9deca",
+    "@dxos/model-factory": "0.3.3-main.ef9deca",
+    "@dxos/network-manager": "0.3.3-main.ef9deca",
+    "@dxos/node-std": "0.3.3-main.ef9deca",
+    "@dxos/protocols": "0.3.3-main.ef9deca",
+    "@dxos/random-access-storage": "0.3.3-main.ef9deca",
+    "@dxos/rpc": "0.3.3-main.ef9deca",
+    "@dxos/teleport": "0.3.3-main.ef9deca",
+    "@dxos/teleport-extension-gossip": "0.3.3-main.ef9deca",
+    "@dxos/teleport-extension-object-sync": "0.3.3-main.ef9deca",
+    "@dxos/teleport-extension-replicator": "0.3.3-main.ef9deca",
+    "@dxos/text-model": "0.3.3-main.ef9deca",
+    "@dxos/timeframe": "0.3.3-main.ef9deca",
+    "@dxos/tracing": "0.3.3-main.ef9deca",
+    "@dxos/typings": "0.3.3-main.ef9deca",
+    "@dxos/util": "0.3.3-main.ef9deca"
   },
   "devDependencies": {
     "fast-check": "~3.3.0",

package/src/db-host/data-service-host.ts

@@ -2,6 +2,7 @@
 // Copyright 2021 DXOS.org
 //
 
+import { UpdateScheduler } from '@dxos/async';
 import { Stream } from '@dxos/codec-protobuf';
 import { Context } from '@dxos/context';
 import { tagMutationsInBatch, type ItemDemuxer, type ItemManager, setMetadataOnObject } from '@dxos/echo-db';
@@ -10,10 +11,20 @@ import { invariant } from '@dxos/invariant';
 import { type PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 import { type DataMessage } from '@dxos/protocols/proto/dxos/echo/feed';
-import { type EchoObjectBatch } from '@dxos/protocols/proto/dxos/echo/object';
+import { type EchoObject, type EchoObjectBatch } from '@dxos/protocols/proto/dxos/echo/object';
 import { EchoEvent, type MutationReceipt, type WriteRequest } from '@dxos/protocols/proto/dxos/echo/service';
 import { ComplexMap } from '@dxos/util';
 
+// After this limit the incremental object updates will be replaced with the full snapshot of the object.
+const MUTATION_LIMIT_PER_OBJECT = 10;
+
+export type DataServiceHostOptions = {
+  /**
+   * @default true
+   */
+  deferEvents?: boolean;
+};
+
 /**
  * Provides methods for DataService for a single space.
  * A DataServiceRouter must be placed before it to route requests to different DataServiceHost instances based on space id.
@@ -30,7 +41,8 @@ export class DataServiceHost {
     private readonly _itemManager: ItemManager,
     private readonly _itemDemuxer: ItemDemuxer,
     private readonly _flush: () => Promise<void>,
-    private readonly _writeStream?: FeedWriter<DataMessage>,
+    private readonly _writeStream: FeedWriter<DataMessage> | undefined,
+    private readonly _opts: DataServiceHostOptions = {},
   ) {}
 
   async open() {}
@@ -39,6 +51,10 @@ export class DataServiceHost {
     await this._ctx.dispose();
   }
 
+  private get _deferEvents(): boolean {
+    return this._opts.deferEvents ?? true;
+  }
+
   /**
    * Real-time subscription to data objects in a space.
    */
@@ -55,8 +71,54 @@ export class DataServiceHost {
         },
       });
 
+      const updateScheduler = new UpdateScheduler(
+        ctx,
+        async () => {
+          flushPendingUpdate();
+        },
+        {
+          maxFrequency: 10,
+        },
+      );
+
+      const pendingUpdates: EchoObject[] = [];
+      const mutationsPerObject = new Map<string, number>();
+
+      const clearPendingUpdates = () => {
+        pendingUpdates.length = 0;
+        mutationsPerObject.clear();
+      };
+
+      const flushPendingUpdate = () => {
+        const stagedEvents: EchoObject[] = [];
+        const objectsWithSnapshots = new Set<string>();
+        for (const [id, count] of mutationsPerObject) {
+          if (count >= MUTATION_LIMIT_PER_OBJECT) {
+            objectsWithSnapshots.add(id);
+            const entity = this._itemManager.entities.get(id);
+            if (entity) {
+              stagedEvents.push(entity.createSnapshot());
+            }
+          }
+        }
+
+        for (const obj of pendingUpdates) {
+          if (!objectsWithSnapshots.has(obj.objectId)) {
+            stagedEvents.push(obj);
+          }
+        }
+
+        next({
+          batch: {
+            objects: stagedEvents,
+          },
+        });
+        clearPendingUpdates();
+      };
+
       // Subscribe to clear events on Epoch processing.
       this._itemDemuxer.snapshot.on(ctx, (snapshot) => {
+        clearPendingUpdates();
         next({
           action: EchoEvent.DatabaseAction.RESET,
           batch: { objects: snapshot.items },
@@ -79,15 +141,34 @@ export class DataServiceHost {
 
         // Assign client tag metadata
         if (clientTag) {
+          flushPendingUpdate();
+
           tagMutationsInBatch(batch, clientTag, 0);
-        }
 
-        next({
-          clientTag,
-          feedKey: message.meta.feedKey,
-          seq: message.meta.seq,
-          batch,
-        });
+          next({
+            clientTag,
+            feedKey: message.meta.feedKey,
+            seq: message.meta.seq,
+            batch,
+          });
+        } else {
+          for (const obj of batch.objects ?? []) {
+            const newCount = (mutationsPerObject.get(obj.objectId) ?? 0) + 1;
+            mutationsPerObject.set(obj.objectId, newCount);
+          }
+
+          for (const obj of batch.objects ?? []) {
+            if ((mutationsPerObject.get(obj.objectId) ?? 0) < MUTATION_LIMIT_PER_OBJECT) {
+              pendingUpdates.push(obj);
+            }
+          }
+
+          if (this._deferEvents) {
+            updateScheduler.trigger();
+          } else {
+            flushPendingUpdate();
+          }
+        }
       });
     });
   }

package/src/db-host/database-host.ts

@@ -8,7 +8,7 @@ import { type ModelFactory } from '@dxos/model-factory';
 import { type DataMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { type EchoSnapshot } from '@dxos/protocols/proto/dxos/echo/snapshot';
 
-import { DataServiceHost } from './data-service-host';
+import { DataServiceHost, type DataServiceHostOptions } from './data-service-host';
 
 /**
  * Database backend that operates on two streams: read and write.
@@ -51,7 +51,13 @@ export class DatabaseHost {
     return this._itemDemuxer.createSnapshot();
   }
 
-  createDataServiceHost() {
-    return new DataServiceHost(this._itemManager, this._itemDemuxer, this._flush, this._outboundStream ?? undefined);
+  createDataServiceHost(opts: DataServiceHostOptions = {}) {
+    return new DataServiceHost(
+      this._itemManager,
+      this._itemDemuxer,
+      this._flush,
+      this._outboundStream ?? undefined,
+      opts,
+    );
   }
 }

package/src/testing/database-test-rig.ts

@@ -114,7 +114,7 @@ export class DatabaseTestPeer {
 
     this.items = new ItemManager(this.modelFactory);
     this.proxy = new DatabaseProxy({
-      service: this.host.createDataServiceHost(),
+      service: this.host.createDataServiceHost({ deferEvents: false }),
       itemManager: this.items,
       spaceKey: this.spaceKey,
     });