@dxos/echo-pipeline 0.3.7 → 0.3.8-main.0ab6c73

package/dist/types/src/db-host/data-service.d.ts

@@ -1,6 +1,6 @@
 import { type Stream } from '@dxos/codec-protobuf';
 import { PublicKey } from '@dxos/keys';
-import { type DataService, type MutationReceipt, type SubscribeRequest, type EchoEvent, type WriteRequest, type FlushRequest } from '@dxos/protocols/proto/dxos/echo/service';
+import { type DataService, type MutationReceipt, type SubscribeRequest, type EchoEvent, type WriteRequest, type FlushRequest, type SyncRepoRequest, type SyncRepoResponse } from '@dxos/protocols/proto/dxos/echo/service';
 import { type DataServiceHost } from './data-service-host';
 export declare class DataServiceSubscriptions {
     private readonly _spaces;
@@ -18,5 +18,7 @@ export declare class DataServiceImpl implements DataService {
     subscribe(request: SubscribeRequest): Stream<EchoEvent>;
     write(request: WriteRequest): Promise<MutationReceipt>;
     flush(request: FlushRequest): Promise<void>;
+    syncRepo(request: SyncRepoRequest): Stream<SyncRepoResponse>;
+    sendSyncMessage(request: SyncRepoRequest): Promise<void>;
 }
 //# sourceMappingURL=data-service.d.ts.map

package/dist/types/src/db-host/data-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"data-service.d.ts","sourceRoot":"","sources":["../../../../src/db-host/data-service.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAGnD,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,EACL,KAAK,WAAW,EAChB,KAAK,eAAe,EACpB,KAAK,gBAAgB,EACrB,KAAK,SAAS,EACd,KAAK,YAAY,EACjB,KAAK,YAAY,EAClB,MAAM,yCAAyC,CAAC;AAGjD,OAAO,EAAE,KAAK,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAG3D,qBAAa,wBAAwB;IACnC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA8D;IAEtF,KAAK;IAIC,aAAa,CAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,eAAe;IAOxD,eAAe,CAAC,QAAQ,EAAE,SAAS;IAOzC,cAAc,CAAC,QAAQ,EAAE,SAAS;CAGnC;AAED;;GAEG;AAEH,qBAAa,eAAgB,YAAW,WAAW;IACrC,OAAO,CAAC,QAAQ,CAAC,cAAc;gBAAd,cAAc,EAAE,wBAAwB;IAErE,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,MAAM,CAAC,SAAS,CAAC;IAOvD,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC;IAQtD,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;CAM5C"}
+{"version":3,"file":"data-service.d.ts","sourceRoot":"","sources":["../../../../src/db-host/data-service.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAGnD,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,EACL,KAAK,WAAW,EAChB,KAAK,eAAe,EACpB,KAAK,gBAAgB,EACrB,KAAK,SAAS,EACd,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,eAAe,EACpB,KAAK,gBAAgB,EACtB,MAAM,yCAAyC,CAAC;AAGjD,OAAO,EAAE,KAAK,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAG3D,qBAAa,wBAAwB;IACnC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA8D;IAEtF,KAAK;IAIC,aAAa,CAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,eAAe;IAOxD,eAAe,CAAC,QAAQ,EAAE,SAAS;IAOzC,cAAc,CAAC,QAAQ,EAAE,SAAS;CAGnC;AAED;;GAEG;AAEH,qBAAa,eAAgB,YAAW,WAAW;IACrC,OAAO,CAAC,QAAQ,CAAC,cAAc;gBAAd,cAAc,EAAE,wBAAwB;IAErE,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,MAAM,CAAC,SAAS,CAAC;IAOvD,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC;IAQtD,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAO3C,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,MAAM,CAAC,gBAAgB,CAAC;IAI5D,eAAe,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;CAGzD"}

package/dist/types/src/space/data-pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"data-pipeline.d.ts","sourceRoot":"","sources":["../../../../src/space/data-pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,EAAiD,MAAM,aAAa,CAAC;AAEnF,OAAO,EACL,KAAK,mBAAmB,EACxB,KAAK,QAAQ,EACb,KAAK,kBAAkB,EAExB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAA2B,WAAW,EAAmB,MAAM,eAAe,CAAC;AAGtF,OAAO,EAAE,KAAK,SAAS,EAAE,MAAM,YAAY,CAAC;AAE5C,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAMxD,OAAO,EAAE,KAAK,UAAU,EAAE,KAAK,KAAK,EAAE,MAAM,6CAA6C,CAAC;AAC1F,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAI5C,OAAO,EAAE,YAAY,EAAE,KAAK,eAAe,EAAE,MAAM,YAAY,CAAC;AAChE,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,MAAM,WAAW,eAAe;IAC9B,YAAY,EAAE,CAAC,KAAK,EAAE,SAAS,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAC;CACvD;AAED,MAAM,MAAM,kBAAkB,GAAG;IAC/B,YAAY,EAAE,YAAY,CAAC;IAC3B,eAAe,EAAE,eAAe,CAAC;IACjC,aAAa,EAAE,aAAa,CAAC;IAC7B,SAAS,EAAE,SAAS,CAAC;IACrB,QAAQ,EAAE,SAAS,CAAC;IACpB,gBAAgB,EAAE,CAAC,OAAO,EAAE,SAAS,KAAK,QAAQ,GAAG,SAAS,CAAC;IAC/D,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAE/B;;OAEG;IACH,iBAAiB,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1D,CAAC;AAiBF;;;;GAIG;AACH,qBAEa,YAAa,YAAW,mBAAmB;IAmC1C,OAAO,CAAC,QAAQ,CAAC,OAAO;IAlCpC,OAAO,CAAC,IAAI,CAAiB;IAC7B,OAAO,CAAC,SAAS,CAAC,CAAuB;IACzC,OAAO,CAAC,gBAAgB,CAAC,CAAwB;IAEjD,OAAO,CAAC,+BAA+B,CAAmB;IAC1D,OAAO,CAAC,OAAO,CAAS;IAExB,OAAO,CAAC,sBAAsB,CAAK;IACnC,OAAO,CAAC,qBAAqB,CAAK;IAClC,OAAO,CAAC,mBAAmB,CAAM;IACjC,OAAO,CAAC,SAAS,CAAC,CAAU;IAG5B,OAAO,CAAC,MAAM,CAA0B;IAGxC,OAAO,CAAC,UAAU,CAA2B;IAEtC,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B,WAAW,EAAG,WAAW,CAAC;IAEjC;;OAEG;IACI,YAAY,CAAC,EAAE,kBAAkB,CAAC,KAAK,CAAC,CAAa;IAE5D;;OAEG;IACI,YAAY,CAAC,EAAE,kBAAkB,CAAC,KAAK,CAAC,CAAa;IAE5D,SAAgB,UAAU,oBAA2B;gBAExB,OAAO,EAAE,kBAAkB;IAExD,IAAI,MAAM,YAET;IAED,IAAI,QAAQ,yBAEX;IAED,IAAI,aAAa,6DAEhB;IAED,kBAAkB,CAAC,SAAS,EAAE,SAAS;IAKjC,iBAAiB,CAAC,UAAU,EAAE,UAAU;IAaxC,IAAI;IAuCJ,KAAK;YAiCG,gBAAgB;IA6D9B,OAAO,CAAC,eAAe;YAST,oBAAoB;YAMpB,UAAU;YAsBV,wBAAwB;YAqBxB,2BAA2B;YA4B3B,aAAa;IAiBrB,kBAAkB,CAAC,SAAS,EAAE,SAAS;IAMvC,WAAW,IAAI,OAAO,CAAC,KAAK,CAAC;IAqB7B,sBAAsB;YAId,MAAM;CAYrB"}
+{"version":3,"file":"data-pipeline.d.ts","sourceRoot":"","sources":["../../../../src/space/data-pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,EAAiD,MAAM,aAAa,CAAC;AAEnF,OAAO,EACL,KAAK,mBAAmB,EACxB,KAAK,QAAQ,EACb,KAAK,kBAAkB,EAExB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAA2B,WAAW,EAAmB,MAAM,eAAe,CAAC;AAGtF,OAAO,EAAE,KAAK,SAAS,EAAE,MAAM,YAAY,CAAC;AAE5C,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAMxD,OAAO,EAAE,KAAK,UAAU,EAAE,KAAK,KAAK,EAAE,MAAM,6CAA6C,CAAC;AAC1F,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAI5C,OAAO,EAAE,YAAY,EAAE,KAAK,eAAe,EAAE,MAAM,YAAY,CAAC;AAChE,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,MAAM,WAAW,eAAe;IAC9B,YAAY,EAAE,CAAC,KAAK,EAAE,SAAS,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAC;CACvD;AAED,MAAM,MAAM,kBAAkB,GAAG;IAC/B,YAAY,EAAE,YAAY,CAAC;IAC3B,eAAe,EAAE,eAAe,CAAC;IACjC,aAAa,EAAE,aAAa,CAAC;IAC7B,SAAS,EAAE,SAAS,CAAC;IACrB,QAAQ,EAAE,SAAS,CAAC;IACpB,gBAAgB,EAAE,CAAC,OAAO,EAAE,SAAS,KAAK,QAAQ,GAAG,SAAS,CAAC;IAC/D,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAE/B;;OAEG;IACH,iBAAiB,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1D,CAAC;AAiBF;;;;GAIG;AACH,qBAEa,YAAa,YAAW,mBAAmB;IAmC1C,OAAO,CAAC,QAAQ,CAAC,OAAO;IAlCpC,OAAO,CAAC,IAAI,CAAiB;IAC7B,OAAO,CAAC,SAAS,CAAC,CAAuB;IACzC,OAAO,CAAC,gBAAgB,CAAC,CAAwB;IAEjD,OAAO,CAAC,+BAA+B,CAAmB;IAC1D,OAAO,CAAC,OAAO,CAAS;IAExB,OAAO,CAAC,sBAAsB,CAAK;IACnC,OAAO,CAAC,qBAAqB,CAAK;IAClC,OAAO,CAAC,mBAAmB,CAAM;IACjC,OAAO,CAAC,SAAS,CAAC,CAAU;IAG5B,OAAO,CAAC,MAAM,CAA0B;IAGxC,OAAO,CAAC,UAAU,CAA2B;IAEtC,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B,WAAW,EAAG,WAAW,CAAC;IAEjC;;OAEG;IACI,YAAY,CAAC,EAAE,kBAAkB,CAAC,KAAK,CAAC,CAAa;IAE5D;;OAEG;IACI,YAAY,CAAC,EAAE,kBAAkB,CAAC,KAAK,CAAC,CAAa;IAE5D,SAAgB,UAAU,oBAA2B;gBAExB,OAAO,EAAE,kBAAkB;IAExD,IAAI,MAAM,YAET;IAED,IAAI,QAAQ,yBAEX;IAED,IAAI,aAAa,6DAEhB;IAED,kBAAkB,CAAC,SAAS,EAAE,SAAS;IAKjC,iBAAiB,CAAC,UAAU,EAAE,UAAU;IAaxC,IAAI;IAuCJ,KAAK;YAiCG,gBAAgB;IA8D9B,OAAO,CAAC,eAAe;YAST,oBAAoB;YAMpB,UAAU;YAsBV,wBAAwB;YAqBxB,2BAA2B;YA4B3B,aAAa;IAiBrB,kBAAkB,CAAC,SAAS,EAAE,SAAS;IAMvC,WAAW,IAAI,OAAO,CAAC,KAAK,CAAC;IAqB7B,sBAAsB;YAId,MAAM;CAYrB"}

package/package.json

@@ -1,17 +1,33 @@
 {
   "name": "@dxos/echo-pipeline",
-  "version": "0.3.7",
+  "version": "0.3.8-main.0ab6c73",
   "description": "ECHO database.",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
   "license": "MIT",
   "author": "info@dxos.org",
-  "main": "dist/lib/node/index.cjs",
-  "browser": {
-    "./dist/lib/node/index.cjs": "./dist/lib/browser/index.mjs",
-    "./testing.js": "./dist/lib/browser/testing/index.mjs"
+  "exports": {
+    ".": {
+      "browser": "./dist/lib/browser/index.mjs",
+      "import": "./dist/lib/browser/index.mjs",
+      "require": "./dist/lib/node/index.cjs",
+      "node": "./dist/lib/node/index.cjs"
+    },
+    "./testing": {
+      "browser": "./dist/lib/browser/testing/index.mjs",
+      "import": "./dist/lib/browser/testing/index.mjs",
+      "require": "./dist/lib/node/testing/index.cjs",
+      "node": "./dist/lib/node/testing/index.cjs"
+    }
   },
   "types": "dist/types/src/index.d.ts",
+  "typesVersions": {
+    "*": {
+      "testing": [
+        "dist/types/src/testing/index.d.ts"
+      ]
+    }
+  },
   "files": [
     "testing.d.ts",
     "testing.js",
@@ -20,36 +36,36 @@
   ],
   "dependencies": {
     "crc-32": "^1.2.2",
-    "@dxos/async": "0.3.7",
-    "@dxos/codec-protobuf": "0.3.7",
-    "@dxos/context": "0.3.7",
-    "@dxos/credentials": "0.3.7",
-    "@dxos/crypto": "0.3.7",
-    "@dxos/debug": "0.3.7",
-    "@dxos/document-model": "0.3.7",
-    "@dxos/echo-db": "0.3.7",
-    "@dxos/feed-store": "0.3.7",
-    "@dxos/hypercore": "0.3.7",
-    "@dxos/invariant": "0.3.7",
-    "@dxos/keyring": "0.3.7",
-    "@dxos/keys": "0.3.7",
-    "@dxos/log": "0.3.7",
-    "@dxos/messaging": "0.3.7",
-    "@dxos/model-factory": "0.3.7",
-    "@dxos/network-manager": "0.3.7",
-    "@dxos/node-std": "0.3.7",
-    "@dxos/protocols": "0.3.7",
-    "@dxos/random-access-storage": "0.3.7",
-    "@dxos/rpc": "0.3.7",
-    "@dxos/teleport": "0.3.7",
-    "@dxos/teleport-extension-gossip": "0.3.7",
-    "@dxos/teleport-extension-object-sync": "0.3.7",
-    "@dxos/teleport-extension-replicator": "0.3.7",
-    "@dxos/text-model": "0.3.7",
-    "@dxos/timeframe": "0.3.7",
-    "@dxos/tracing": "0.3.7",
-    "@dxos/typings": "0.3.7",
-    "@dxos/util": "0.3.7"
+    "@dxos/async": "0.3.8-main.0ab6c73",
+    "@dxos/codec-protobuf": "0.3.8-main.0ab6c73",
+    "@dxos/context": "0.3.8-main.0ab6c73",
+    "@dxos/crypto": "0.3.8-main.0ab6c73",
+    "@dxos/credentials": "0.3.8-main.0ab6c73",
+    "@dxos/debug": "0.3.8-main.0ab6c73",
+    "@dxos/document-model": "0.3.8-main.0ab6c73",
+    "@dxos/echo-db": "0.3.8-main.0ab6c73",
+    "@dxos/feed-store": "0.3.8-main.0ab6c73",
+    "@dxos/hypercore": "0.3.8-main.0ab6c73",
+    "@dxos/invariant": "0.3.8-main.0ab6c73",
+    "@dxos/keyring": "0.3.8-main.0ab6c73",
+    "@dxos/keys": "0.3.8-main.0ab6c73",
+    "@dxos/log": "0.3.8-main.0ab6c73",
+    "@dxos/messaging": "0.3.8-main.0ab6c73",
+    "@dxos/network-manager": "0.3.8-main.0ab6c73",
+    "@dxos/model-factory": "0.3.8-main.0ab6c73",
+    "@dxos/protocols": "0.3.8-main.0ab6c73",
+    "@dxos/node-std": "0.3.8-main.0ab6c73",
+    "@dxos/random-access-storage": "0.3.8-main.0ab6c73",
+    "@dxos/rpc": "0.3.8-main.0ab6c73",
+    "@dxos/teleport": "0.3.8-main.0ab6c73",
+    "@dxos/teleport-extension-gossip": "0.3.8-main.0ab6c73",
+    "@dxos/teleport-extension-object-sync": "0.3.8-main.0ab6c73",
+    "@dxos/text-model": "0.3.8-main.0ab6c73",
+    "@dxos/timeframe": "0.3.8-main.0ab6c73",
+    "@dxos/teleport-extension-replicator": "0.3.8-main.0ab6c73",
+    "@dxos/tracing": "0.3.8-main.0ab6c73",
+    "@dxos/typings": "0.3.8-main.0ab6c73",
+    "@dxos/util": "0.3.8-main.0ab6c73"
   },
   "devDependencies": {
     "fast-check": "~3.3.0",

package/src/db-host/data-service-host.ts

@@ -12,7 +12,13 @@ import { type PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 import { type DataMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { type EchoObject, type EchoObjectBatch } from '@dxos/protocols/proto/dxos/echo/object';
-import { EchoEvent, type MutationReceipt, type WriteRequest } from '@dxos/protocols/proto/dxos/echo/service';
+import {
+  EchoEvent,
+  type SyncRepoRequest,
+  type MutationReceipt,
+  type WriteRequest,
+  type SyncRepoResponse,
+} from '@dxos/protocols/proto/dxos/echo/service';
 import { ComplexMap } from '@dxos/util';
 
 // After this limit the incremental object updates will be replaced with the full snapshot of the object.
@@ -198,6 +204,14 @@ export class DataServiceHost {
   async flush(): Promise<void> {
     await this._flush();
   }
+
+  syncRepo(request: SyncRepoRequest): Stream<SyncRepoResponse> {
+    throw new Error('Method not implemented.');
+  }
+
+  sendSyncMessage(request: SyncRepoRequest): Promise<void> {
+    throw new Error('Method not implemented.');
+  }
 }
 
 const createDataMessage = (batch: EchoObjectBatch) => ({

package/src/db-host/data-service.ts

@@ -14,6 +14,8 @@ import {
   type EchoEvent,
   type WriteRequest,
   type FlushRequest,
+  type SyncRepoRequest,
+  type SyncRepoResponse,
 } from '@dxos/protocols/proto/dxos/echo/service';
 import { ComplexMap } from '@dxos/util';
 
@@ -74,4 +76,12 @@ export class DataServiceImpl implements DataService {
       this._subscriptions.getDataService(request.spaceKey) ?? raise(new Error(`space not found: ${request.spaceKey}`));
     return host.flush();
   }
+
+  syncRepo(request: SyncRepoRequest): Stream<SyncRepoResponse> {
+    throw new Error('Method not implemented.');
+  }
+
+  sendSyncMessage(request: SyncRepoRequest): Promise<void> {
+    throw new Error('Method not implemented.');
+  }
 }

package/src/space/data-pipeline.ts

@@ -217,6 +217,7 @@ export class DataPipeline implements CredentialProcessor {
       await waitForOneEpoch;
     }
 
+    // CPU bottleneck control.
     let messageCounter = 0;
 
     invariant(this._pipeline, 'Pipeline is not initialized.');
@@ -263,10 +264,10 @@ export class DataPipeline implements CredentialProcessor {
 
       span.end();
 
-      if (++messageCounter > 1_000) {
+      if (++messageCounter > 100) {
         messageCounter = 0;
         // Allow other tasks to process.
-        await sleep(1);
+        await idle(1_000);
       }
     }
   }
@@ -418,3 +419,39 @@ export class DataPipeline implements CredentialProcessor {
     await this._params.metadataStore.flush();
   }
 }
+
+/**
+ * Waits up to `timeout` ms for the browser to be idle.
+ */
+const idle = async (timeout?: number) => {
+  if (!('scheduler' in globalThis && typeof (globalThis as any).scheduler.postTask === 'function')) {
+    await sleep(1);
+    return;
+  }
+
+  await new Promise<void>((resolve) => {
+    // const beginTime = performance.now();
+    const cleanup = () => {
+      clearTimeout(timer);
+      controller.abort();
+      // log.warn('yielded for', { ms: performance.now() - beginTime });
+    };
+
+    const controller = new AbortController();
+
+    void (globalThis as any).scheduler
+      .postTask(
+        () => {
+          cleanup();
+          resolve();
+        },
+        { priority: 'background', signal: controller.signal },
+      )
+      .catch(() => {});
+
+    const timer = setTimeout(() => {
+      cleanup();
+      resolve();
+    }, timeout);
+  });
+};