@dxos/echo-pipeline 0.7.4 → 0.7.5-feature-compute.4d9d99a

package/dist/lib/browser/index.mjs

@@ -21,7 +21,7 @@ import {
   mapTimeframeToFeedIndexes,
   startAfter,
   valueEncoding
-} from "./chunk-LZK5YFYE.mjs";
+} from "./chunk-QBMTPEMY.mjs";
 
 // packages/core/echo/echo-pipeline/src/db-host/data-service.ts
 import { UpdateScheduler as UpdateScheduler2 } from "@dxos/async";
@@ -310,7 +310,7 @@ var CollectionSynchronizer = class extends Resource2 {
         return;
       }
       for (const [collectionId, state] of this._perCollectionStates.entries()) {
-        if (this._shouldSyncCollection(collectionId, peerId)) {
+        if (this._activeCollections.has(collectionId) && this._shouldSyncCollection(collectionId, peerId)) {
           state.interestedPeers.add(peerId);
           state.lastQueried.set(peerId, Date.now());
           this._queryCollectionState(collectionId, peerId);
@@ -352,11 +352,18 @@ var CollectionSynchronizer = class extends Resource2 {
     });
     validateCollectionState(state);
     const perCollectionState = this._getOrCreatePerCollectionState(collectionId);
-    perCollectionState.remoteStates.set(peerId, state);
-    this.remoteStateUpdated.emit({
-      peerId,
-      collectionId
-    });
+    const existingState = perCollectionState.remoteStates.get(peerId) ?? {
+      documents: {}
+    };
+    const diff = diffCollectionState(existingState, state);
+    if (diff.missingOnLocal.length > 0 || diff.different.length > 0) {
+      perCollectionState.remoteStates.set(peerId, state);
+      this.remoteStateUpdated.emit({
+        peerId,
+        collectionId,
+        newDocsAppeared: diff.missingOnLocal.length > 0
+      });
+    }
   }
   _getOrCreatePerCollectionState(collectionId) {
     return defaultMap(this._perCollectionStates, collectionId, () => ({
@@ -677,6 +684,12 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
     }
     this._params.monitor?.recordMessageReceived(message);
   }
+  onConnectionAuthScopeChanged(peer) {
+    const entry = this._connections.get(peer);
+    if (entry) {
+      this._onConnectionAuthScopeChanged(entry.connection);
+    }
+  }
   /**
   * Trigger doc-synchronizer shared documents set recalculation. Happens on peer-candidate.
   * TODO(y): replace with a proper API call when sharePolicy update becomes supported by automerge-repo
@@ -686,14 +699,14 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
       peerId: connection.peerId
     }, {
       F: __dxlog_file3,
-      L: 254,
+      L: 261,
       S: this,
       C: (f, a) => f(...a)
     });
     const entry = this._connections.get(connection.peerId);
     invariant2(entry, void 0, {
       F: __dxlog_file3,
-      L: 256,
+      L: 263,
       S: this,
       A: [
         "entry",
@@ -710,14 +723,14 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
       peerId: connection.peerId
     }, {
       F: __dxlog_file3,
-      L: 262,
+      L: 269,
       S: this,
       C: (f, a) => f(...a)
     });
     const entry = this._connections.get(connection.peerId);
     invariant2(entry, void 0, {
       F: __dxlog_file3,
-      L: 264,
+      L: 271,
       S: this,
       A: [
         "entry",
@@ -731,13 +744,13 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
     this._params.monitor?.recordPeerDisconnected(connection.peerId);
     void entry.reader.cancel().catch((err) => log3.catch(err, void 0, {
       F: __dxlog_file3,
-      L: 270,
+      L: 277,
       S: this,
       C: (f, a) => f(...a)
     }));
     void entry.writer.abort().catch((err) => log3.catch(err, void 0, {
       F: __dxlog_file3,
-      L: 271,
+      L: 278,
       S: this,
       C: (f, a) => f(...a)
     }));
@@ -905,7 +918,7 @@ function _ts_decorate2(decorators, target, key, desc) {
 }
 var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/automerge-host.ts";
 var AutomergeHost = class extends Resource4 {
-  constructor({ db, indexMetadataStore, dataMonitor, peerIdProvider }) {
+  constructor({ db, indexMetadataStore, dataMonitor, peerIdProvider, getSpaceKeyByRootDocumentId }) {
     super();
     this._collectionSynchronizer = new CollectionSynchronizer({
       queryCollectionState: this._queryCollectionState.bind(this),
@@ -934,6 +947,7 @@ var AutomergeHost = class extends Resource4 {
     });
     this._indexMetadataStore = indexMetadataStore;
     this._peerIdProvider = peerIdProvider;
+    this._getSpaceKeyByRootDocumentId = getSpaceKeyByRootDocumentId;
   }
   async _open() {
     this._peerId = `host-${this._peerIdProvider?.() ?? PublicKey.random().toHex()}`;
@@ -947,13 +961,22 @@ var AutomergeHost = class extends Resource4 {
         this._echoNetworkAdapter
       ]
     });
-    Event2.wrap(this._echoNetworkAdapter, "peer-candidate").on(this._ctx, (e) => this._onPeerConnected(e.peerId));
-    Event2.wrap(this._echoNetworkAdapter, "peer-disconnected").on(this._ctx, (e) => this._onPeerDisconnected(e.peerId));
-    this._collectionSynchronizer.remoteStateUpdated.on(this._ctx, ({ collectionId, peerId }) => {
+    let updatingAuthScope = false;
+    Event2.wrap(this._echoNetworkAdapter, "peer-candidate").on(this._ctx, (e) => !updatingAuthScope && this._onPeerConnected(e.peerId));
+    Event2.wrap(this._echoNetworkAdapter, "peer-disconnected").on(this._ctx, (e) => !updatingAuthScope && this._onPeerDisconnected(e.peerId));
+    this._collectionSynchronizer.remoteStateUpdated.on(this._ctx, ({ collectionId, peerId, newDocsAppeared }) => {
       this._onRemoteCollectionStateUpdated(collectionId, peerId);
       this.collectionStateUpdated.emit({
         collectionId
       });
+      if (newDocsAppeared) {
+        updatingAuthScope = true;
+        try {
+          this._echoNetworkAdapter.onConnectionAuthScopeChanged(peerId);
+        } finally {
+          updatingAuthScope = false;
+        }
+      }
     });
     await this._echoNetworkAdapter.open();
     await this._collectionSynchronizer.open();
@@ -1036,7 +1059,7 @@ var AutomergeHost = class extends Resource4 {
       await Promise.all(headsToWait.map(async (entry, index) => {
         const handle = await this.loadDoc(Context.default(void 0, {
           F: __dxlog_file4,
-          L: 239
+          L: 264
         }), entry.documentId);
         await waitForHeads(handle, entry.heads);
       }));
@@ -1049,7 +1072,7 @@ var AutomergeHost = class extends Resource4 {
         documentId
       }, {
         F: __dxlog_file4,
-        L: 251,
+        L: 276,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -1065,7 +1088,7 @@ var AutomergeHost = class extends Resource4 {
           documentId
         }, {
           F: __dxlog_file4,
-          L: 255,
+          L: 280,
           S: this,
           C: (f, a) => f(...a)
         });
@@ -1074,7 +1097,7 @@ var AutomergeHost = class extends Resource4 {
       const doc = handle.docSync();
       invariant3(doc, void 0, {
         F: __dxlog_file4,
-        L: 260,
+        L: 285,
         S: this,
         A: [
           "doc",
@@ -1088,7 +1111,7 @@ var AutomergeHost = class extends Resource4 {
     }
     log4.info("done re-indexing heads", void 0, {
       F: __dxlog_file4,
-      L: 267,
+      L: 292,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -1172,14 +1195,17 @@ var AutomergeHost = class extends Resource4 {
   }
   async _getContainingSpaceForDocument(documentId) {
     const doc = this._repo.handles[documentId]?.docSync();
-    if (!doc) {
-      return null;
+    if (doc) {
+      const spaceKeyHex = getSpaceKeyFromDoc(doc);
+      if (spaceKeyHex) {
+        return PublicKey.from(spaceKeyHex);
+      }
     }
-    const spaceKeyHex = getSpaceKeyFromDoc(doc);
-    if (!spaceKeyHex) {
-      return null;
+    const rootDocSpaceKey = this._getSpaceKeyByRootDocumentId?.(documentId);
+    if (rootDocSpaceKey) {
+      return rootDocSpaceKey;
     }
-    return PublicKey.from(spaceKeyHex);
+    return null;
   }
   /**
   * Flush documents to disk.
@@ -1293,11 +1319,14 @@ var AutomergeHost = class extends Resource4 {
     if (toReplicate.length === 0) {
       return;
     }
-    log4.info("replication documents after collection sync", {
+    log4.info("replicating documents after collection sync", {
+      collectionId,
+      peerId,
+      toReplicate,
       count: toReplicate.length
     }, {
       F: __dxlog_file4,
-      L: 499,
+      L: 531,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -1364,7 +1393,7 @@ var changeIsPresentInDoc = (doc, changeHash) => {
 var decodeCollectionState = (state) => {
   invariant3(typeof state === "object" && state !== null, "Invalid state", {
     F: __dxlog_file4,
-    L: 557,
+    L: 592,
     S: void 0,
     A: [
       "typeof state === 'object' && state !== null",
@@ -1656,10 +1685,10 @@ var MeshEchoReplicator = class {
               documentId: params.documentId,
               peerId: connection.peerId
             });
-            log6("document not found locally for share policy check, accepting the remote document", {
+            log6("document not found locally for share policy check", {
               peerId: connection.peerId,
               documentId: params.documentId,
-              remoteDocumentExists
+              acceptDocument: remoteDocumentExists
             }, {
               F: __dxlog_file7,
               L: 91,
@@ -2923,7 +2952,7 @@ var INDEXER_CONFIG = {
   ]
 };
 var EchoHost = class extends Resource9 {
-  constructor({ kv, peerIdProvider }) {
+  constructor({ kv, peerIdProvider, getSpaceKeyByRootDocumentId }) {
     super();
     this._spaceStateManager = new SpaceStateManager();
     this._indexMetadataStore = new IndexMetadataStore({
@@ -2934,7 +2963,8 @@ var EchoHost = class extends Resource9 {
       db: kv,
       dataMonitor: this._echoDataMonitor,
       indexMetadataStore: this._indexMetadataStore,
-      peerIdProvider
+      peerIdProvider,
+      getSpaceKeyByRootDocumentId
     });
     this._indexer = new Indexer({
       db: kv,
@@ -3059,7 +3089,7 @@ var EchoHost = class extends Resource9 {
   async createSpaceRoot(spaceKey) {
     invariant11(this._lifecycleState === LifecycleState4.OPEN, void 0, {
       F: __dxlog_file15,
-      L: 217,
+      L: 220,
       S: this,
       A: [
         "this._lifecycleState === LifecycleState.OPEN",
@@ -3087,7 +3117,7 @@ var EchoHost = class extends Resource9 {
   async openSpaceRoot(spaceId, automergeUrl) {
     invariant11(this._lifecycleState === LifecycleState4.OPEN, void 0, {
       F: __dxlog_file15,
-      L: 236,
+      L: 239,
       S: this,
       A: [
         "this._lifecycleState === LifecycleState.OPEN",
@@ -3117,16 +3147,73 @@ var EchoHost = class extends Resource9 {
 
 // packages/core/echo/echo-pipeline/src/edge/echo-edge-replicator.ts
 import { Mutex, scheduleTask as scheduleTask2, scheduleMicroTask } from "@dxos/async";
-import * as A5 from "@dxos/automerge/automerge";
 import { cbor as cbor2 } from "@dxos/automerge/automerge-repo";
-import { Context as Context6, Resource as Resource10 } from "@dxos/context";
+import { Context as Context6, Resource as Resource11 } from "@dxos/context";
 import { randomUUID } from "@dxos/crypto";
 import { invariant as invariant12 } from "@dxos/invariant";
-import { log as log11 } from "@dxos/log";
+import { log as log12 } from "@dxos/log";
 import { EdgeService } from "@dxos/protocols";
 import { buf } from "@dxos/protocols/buf";
 import { MessageSchema as RouterMessageSchema } from "@dxos/protocols/buf/dxos/edge/messenger_pb";
 import { bufferToArray } from "@dxos/util";
+
+// packages/core/echo/echo-pipeline/src/edge/inflight-request-limiter.ts
+import { Trigger as Trigger2 } from "@dxos/async";
+import { Resource as Resource10 } from "@dxos/context";
+import { log as log11 } from "@dxos/log";
+var __dxlog_file16 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/edge/inflight-request-limiter.ts";
+var InflightRequestLimiter = class extends Resource10 {
+  constructor(_config) {
+    super();
+    this._config = _config;
+    this._inflightRequestBalance = 0;
+    this._requestBarrier = new Trigger2();
+  }
+  async _open() {
+    this._inflightRequestBalance = 0;
+    this._requestBarrier.reset();
+    this._requestBarrier.wake();
+  }
+  async _close() {
+    this._inflightRequestBalance = 0;
+    this._requestBarrier.throw(new Error("Rate limiter closed."));
+    clearTimeout(this._resetBalanceTimeout);
+  }
+  async rateLimit(message) {
+    if (message.type !== "sync") {
+      return;
+    }
+    while (this._inflightRequestBalance >= this._config.maxInflightRequests) {
+      await this._requestBarrier.wait();
+    }
+    this._inflightRequestBalance++;
+    if (this._inflightRequestBalance === this._config.maxInflightRequests) {
+      this._requestBarrier.reset();
+      this._resetBalanceTimeout = setTimeout(() => {
+        log11.warn("Request balance has not changed during specified timeout, resetting request limiter.", void 0, {
+          F: __dxlog_file16,
+          L: 52,
+          S: this,
+          C: (f, a) => f(...a)
+        });
+        this._inflightRequestBalance = 0;
+        this._requestBarrier.wake();
+      }, this._config.resetBalanceTimeoutMs);
+    }
+  }
+  handleResponse(message) {
+    if (message.type !== "sync") {
+      return;
+    }
+    this._inflightRequestBalance--;
+    if (this._inflightRequestBalance + 1 === this._config.maxInflightRequests) {
+      this._requestBarrier.wake();
+      clearInterval(this._resetBalanceTimeout);
+    }
+  }
+};
+
+// packages/core/echo/echo-pipeline/src/edge/echo-edge-replicator.ts
 function _using_ctx() {
   var _disposeSuppressedError = typeof SuppressedError === "function" ? SuppressedError : function(error, suppressed) {
     var err = new Error();
@@ -3189,7 +3276,7 @@ function _using_ctx() {
     }
   };
 }
-var __dxlog_file16 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/edge/echo-edge-replicator.ts";
+var __dxlog_file17 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/edge/echo-edge-replicator.ts";
 var INITIAL_RESTART_DELAY = 500;
 var RESTART_DELAY_JITTER = 250;
 var MAX_RESTART_DELAY = 5e3;
@@ -3205,18 +3292,18 @@ var EchoEdgeReplicator = class {
     this._sharePolicyEnabled = !disableSharePolicy;
   }
   async connect(context) {
-    log11.info("connect", {
+    log12.info("connect", {
       peerId: context.peerId,
       connectedSpaces: this._connectedSpaces.size
     }, {
-      F: __dxlog_file16,
+      F: __dxlog_file17,
       L: 60,
       S: this,
       C: (f, a) => f(...a)
     });
     this._context = context;
     this._ctx = Context6.default(void 0, {
-      F: __dxlog_file16,
+      F: __dxlog_file17,
       L: 63
     });
     this._ctx.onDispose(this._edgeConnection.onReconnected(() => {
@@ -3295,7 +3382,7 @@ var EchoEdgeReplicator = class {
   }
   async _openConnection(spaceId, reconnects = 0) {
     invariant12(this._context, void 0, {
-      F: __dxlog_file16,
+      F: __dxlog_file17,
       L: 124,
       S: this,
       A: [
@@ -3304,7 +3391,7 @@ var EchoEdgeReplicator = class {
       ]
     });
     invariant12(!this._connections.has(spaceId), void 0, {
-      F: __dxlog_file16,
+      F: __dxlog_file17,
       L: 125,
       S: this,
       A: [
@@ -3329,12 +3416,12 @@ var EchoEdgeReplicator = class {
           return;
         }
         const restartDelay = Math.min(MAX_RESTART_DELAY, INITIAL_RESTART_DELAY * reconnects) + Math.random() * RESTART_DELAY_JITTER;
-        log11.info("connection restart scheduled", {
+        log12.info("connection restart scheduled", {
           spaceId,
           reconnects,
           restartDelay
         }, {
-          F: __dxlog_file16,
+          F: __dxlog_file17,
           L: 148,
           S: this,
           C: (f, a) => f(...a)
@@ -3366,10 +3453,16 @@ var EchoEdgeReplicator = class {
     await connection.open();
   }
 };
-var EdgeReplicatorConnection = class extends Resource10 {
+var MAX_INFLIGHT_REQUESTS = 5;
+var MAX_RATE_LIMIT_WAIT_TIME_MS = 3e3;
+var EdgeReplicatorConnection = class extends Resource11 {
   constructor({ edgeConnection, spaceId, context, sharedPolicyEnabled, onRemoteConnected, onRemoteDisconnected, onRestartRequested }) {
     super();
     this._remotePeerId = null;
+    this._requestLimiter = new InflightRequestLimiter({
+      maxInflightRequests: MAX_INFLIGHT_REQUESTS,
+      resetBalanceTimeoutMs: MAX_RATE_LIMIT_WAIT_TIME_MS
+    });
     this._edgeConnection = edgeConnection;
     this._spaceId = spaceId;
     this._context = context;
@@ -3386,36 +3479,39 @@ var EdgeReplicatorConnection = class extends Resource10 {
     });
     this.writable = new WritableStream({
       write: async (message, controller) => {
+        await this._requestLimiter.rateLimit(message);
         await this._sendMessage(message);
       }
     });
   }
   async _open(ctx) {
-    log11("open", void 0, {
-      F: __dxlog_file16,
-      L: 242,
+    log12("open", void 0, {
+      F: __dxlog_file17,
+      L: 251,
       S: this,
       C: (f, a) => f(...a)
     });
+    await this._requestLimiter.open();
     this._ctx.onDispose(this._edgeConnection.onMessage((msg) => {
       this._onMessage(msg);
     }));
     await this._onRemoteConnected();
   }
   async _close() {
-    log11("close", void 0, {
-      F: __dxlog_file16,
-      L: 254,
+    log12("close", void 0, {
+      F: __dxlog_file17,
+      L: 266,
       S: this,
       C: (f, a) => f(...a)
     });
     this._readableStreamController.close();
+    await this._requestLimiter.close();
     await this._onRemoteDisconnected();
   }
   get peerId() {
     invariant12(this._remotePeerId, "Not connected", {
-      F: __dxlog_file16,
-      L: 260,
+      F: __dxlog_file17,
+      L: 275,
       S: this,
       A: [
         "this._remotePeerId",
@@ -3434,12 +3530,13 @@ var EdgeReplicatorConnection = class extends Resource10 {
         documentId: params.documentId,
         peerId: this._remotePeerId
       });
-      log11.info("document not found locally for share policy check, accepting the remote document", {
+      log12.verbose("edge-replicator document not found locally for share policy check", {
         documentId: params.documentId,
-        remoteDocumentExists
+        acceptDocument: remoteDocumentExists,
+        remoteId: this._remotePeerId
       }, {
-        F: __dxlog_file16,
-        L: 275,
+        F: __dxlog_file17,
+        L: 290,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -3452,23 +3549,20 @@ var EdgeReplicatorConnection = class extends Resource10 {
       return true;
     }
     const spaceId = getSpaceIdFromCollectionId(params.collectionId);
-    return spaceId === this._spaceId;
+    return spaceId === this._spaceId && params.collectionId.split(":").length === 3;
   }
   _onMessage(message) {
     if (message.serviceId !== this._targetServiceId) {
       return;
     }
     const payload = cbor2.decode(message.payload.value);
-    log11("recv", () => {
-      const decodedData = payload.type === "sync" && payload.data ? A5.decodeSyncMessage(payload.data) : payload.type === "collection-state" ? payload.state : payload;
-      return {
-        from: message.serviceId,
-        type: payload.type,
-        decodedData
-      };
+    log12.verbose("edge replicator receive", {
+      type: payload.type,
+      documentId: payload.type === "sync" && payload.documentId,
+      remoteId: this._remotePeerId
     }, {
-      F: __dxlog_file16,
-      L: 302,
+      F: __dxlog_file17,
+      L: 319,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -3480,18 +3574,18 @@ var EdgeReplicatorConnection = class extends Resource10 {
       this._onRestartRequested();
       return;
     }
+    this._requestLimiter.handleResponse(message);
     this._readableStreamController.enqueue(message);
   }
   async _sendMessage(message) {
     message.targetId = this._targetServiceId;
-    log11("send", {
+    log12.verbose("edge replicator send", {
       type: message.type,
-      senderId: message.senderId,
-      targetId: message.targetId,
-      documentId: message.documentId
+      documentId: message.type === "sync" && message.documentId,
+      remoteId: this._remotePeerId
     }, {
-      F: __dxlog_file16,
-      L: 332,
+      F: __dxlog_file17,
+      L: 348,
       S: this,
       C: (f, a) => f(...a)
     });