@dxos/echo-pipeline 0.7.4 → 0.7.5-main.9cb18ac

package/dist/lib/browser/index.mjs

@@ -21,7 +21,7 @@ import {
   mapTimeframeToFeedIndexes,
   startAfter,
   valueEncoding
-} from "./chunk-LZK5YFYE.mjs";
+} from "./chunk-KY5AZOEF.mjs";
 
 // packages/core/echo/echo-pipeline/src/db-host/data-service.ts
 import { UpdateScheduler as UpdateScheduler2 } from "@dxos/async";
@@ -310,7 +310,7 @@ var CollectionSynchronizer = class extends Resource2 {
         return;
       }
       for (const [collectionId, state] of this._perCollectionStates.entries()) {
-        if (this._shouldSyncCollection(collectionId, peerId)) {
+        if (this._activeCollections.has(collectionId) && this._shouldSyncCollection(collectionId, peerId)) {
           state.interestedPeers.add(peerId);
           state.lastQueried.set(peerId, Date.now());
           this._queryCollectionState(collectionId, peerId);
@@ -352,11 +352,18 @@ var CollectionSynchronizer = class extends Resource2 {
     });
     validateCollectionState(state);
     const perCollectionState = this._getOrCreatePerCollectionState(collectionId);
-    perCollectionState.remoteStates.set(peerId, state);
-    this.remoteStateUpdated.emit({
-      peerId,
-      collectionId
-    });
+    const existingState = perCollectionState.remoteStates.get(peerId) ?? {
+      documents: {}
+    };
+    const diff = diffCollectionState(existingState, state);
+    if (diff.missingOnLocal.length > 0 || diff.different.length > 0) {
+      perCollectionState.remoteStates.set(peerId, state);
+      this.remoteStateUpdated.emit({
+        peerId,
+        collectionId,
+        newDocsAppeared: diff.missingOnLocal.length > 0
+      });
+    }
   }
   _getOrCreatePerCollectionState(collectionId) {
     return defaultMap(this._perCollectionStates, collectionId, () => ({
@@ -677,6 +684,12 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
     }
     this._params.monitor?.recordMessageReceived(message);
   }
+  onConnectionAuthScopeChanged(peer) {
+    const entry = this._connections.get(peer);
+    if (entry) {
+      this._onConnectionAuthScopeChanged(entry.connection);
+    }
+  }
   /**
   * Trigger doc-synchronizer shared documents set recalculation. Happens on peer-candidate.
   * TODO(y): replace with a proper API call when sharePolicy update becomes supported by automerge-repo
@@ -686,14 +699,14 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
       peerId: connection.peerId
     }, {
       F: __dxlog_file3,
-      L: 254,
+      L: 261,
       S: this,
       C: (f, a) => f(...a)
     });
     const entry = this._connections.get(connection.peerId);
     invariant2(entry, void 0, {
       F: __dxlog_file3,
-      L: 256,
+      L: 263,
       S: this,
       A: [
         "entry",
@@ -710,14 +723,14 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
       peerId: connection.peerId
     }, {
       F: __dxlog_file3,
-      L: 262,
+      L: 269,
       S: this,
       C: (f, a) => f(...a)
     });
     const entry = this._connections.get(connection.peerId);
     invariant2(entry, void 0, {
       F: __dxlog_file3,
-      L: 264,
+      L: 271,
       S: this,
       A: [
         "entry",
@@ -731,13 +744,13 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
     this._params.monitor?.recordPeerDisconnected(connection.peerId);
     void entry.reader.cancel().catch((err) => log3.catch(err, void 0, {
       F: __dxlog_file3,
-      L: 270,
+      L: 277,
       S: this,
       C: (f, a) => f(...a)
     }));
     void entry.writer.abort().catch((err) => log3.catch(err, void 0, {
       F: __dxlog_file3,
-      L: 271,
+      L: 278,
       S: this,
       C: (f, a) => f(...a)
     }));
@@ -905,7 +918,7 @@ function _ts_decorate2(decorators, target, key, desc) {
 }
 var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/automerge-host.ts";
 var AutomergeHost = class extends Resource4 {
-  constructor({ db, indexMetadataStore, dataMonitor, peerIdProvider }) {
+  constructor({ db, indexMetadataStore, dataMonitor, peerIdProvider, getSpaceKeyByRootDocumentId }) {
     super();
     this._collectionSynchronizer = new CollectionSynchronizer({
       queryCollectionState: this._queryCollectionState.bind(this),
@@ -934,6 +947,7 @@ var AutomergeHost = class extends Resource4 {
     });
     this._indexMetadataStore = indexMetadataStore;
     this._peerIdProvider = peerIdProvider;
+    this._getSpaceKeyByRootDocumentId = getSpaceKeyByRootDocumentId;
   }
   async _open() {
     this._peerId = `host-${this._peerIdProvider?.() ?? PublicKey.random().toHex()}`;
@@ -947,13 +961,22 @@ var AutomergeHost = class extends Resource4 {
         this._echoNetworkAdapter
       ]
     });
-    Event2.wrap(this._echoNetworkAdapter, "peer-candidate").on(this._ctx, (e) => this._onPeerConnected(e.peerId));
-    Event2.wrap(this._echoNetworkAdapter, "peer-disconnected").on(this._ctx, (e) => this._onPeerDisconnected(e.peerId));
-    this._collectionSynchronizer.remoteStateUpdated.on(this._ctx, ({ collectionId, peerId }) => {
+    let updatingAuthScope = false;
+    Event2.wrap(this._echoNetworkAdapter, "peer-candidate").on(this._ctx, (e) => !updatingAuthScope && this._onPeerConnected(e.peerId));
+    Event2.wrap(this._echoNetworkAdapter, "peer-disconnected").on(this._ctx, (e) => !updatingAuthScope && this._onPeerDisconnected(e.peerId));
+    this._collectionSynchronizer.remoteStateUpdated.on(this._ctx, ({ collectionId, peerId, newDocsAppeared }) => {
       this._onRemoteCollectionStateUpdated(collectionId, peerId);
       this.collectionStateUpdated.emit({
         collectionId
       });
+      if (newDocsAppeared) {
+        updatingAuthScope = true;
+        try {
+          this._echoNetworkAdapter.onConnectionAuthScopeChanged(peerId);
+        } finally {
+          updatingAuthScope = false;
+        }
+      }
     });
     await this._echoNetworkAdapter.open();
     await this._collectionSynchronizer.open();
@@ -1036,7 +1059,7 @@ var AutomergeHost = class extends Resource4 {
       await Promise.all(headsToWait.map(async (entry, index) => {
         const handle = await this.loadDoc(Context.default(void 0, {
           F: __dxlog_file4,
-          L: 239
+          L: 264
         }), entry.documentId);
         await waitForHeads(handle, entry.heads);
       }));
@@ -1049,7 +1072,7 @@ var AutomergeHost = class extends Resource4 {
         documentId
       }, {
         F: __dxlog_file4,
-        L: 251,
+        L: 276,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -1065,7 +1088,7 @@ var AutomergeHost = class extends Resource4 {
           documentId
         }, {
           F: __dxlog_file4,
-          L: 255,
+          L: 280,
           S: this,
           C: (f, a) => f(...a)
         });
@@ -1074,7 +1097,7 @@ var AutomergeHost = class extends Resource4 {
       const doc = handle.docSync();
       invariant3(doc, void 0, {
         F: __dxlog_file4,
-        L: 260,
+        L: 285,
         S: this,
         A: [
           "doc",
@@ -1088,7 +1111,7 @@ var AutomergeHost = class extends Resource4 {
     }
     log4.info("done re-indexing heads", void 0, {
       F: __dxlog_file4,
-      L: 267,
+      L: 292,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -1172,14 +1195,17 @@ var AutomergeHost = class extends Resource4 {
   }
   async _getContainingSpaceForDocument(documentId) {
     const doc = this._repo.handles[documentId]?.docSync();
-    if (!doc) {
-      return null;
+    if (doc) {
+      const spaceKeyHex = getSpaceKeyFromDoc(doc);
+      if (spaceKeyHex) {
+        return PublicKey.from(spaceKeyHex);
+      }
     }
-    const spaceKeyHex = getSpaceKeyFromDoc(doc);
-    if (!spaceKeyHex) {
-      return null;
+    const rootDocSpaceKey = this._getSpaceKeyByRootDocumentId?.(documentId);
+    if (rootDocSpaceKey) {
+      return rootDocSpaceKey;
     }
-    return PublicKey.from(spaceKeyHex);
+    return null;
   }
   /**
   * Flush documents to disk.
@@ -1293,11 +1319,14 @@ var AutomergeHost = class extends Resource4 {
     if (toReplicate.length === 0) {
       return;
     }
-    log4.info("replication documents after collection sync", {
+    log4.info("replicating documents after collection sync", {
+      collectionId,
+      peerId,
+      toReplicate,
       count: toReplicate.length
     }, {
       F: __dxlog_file4,
-      L: 499,
+      L: 531,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -1364,7 +1393,7 @@ var changeIsPresentInDoc = (doc, changeHash) => {
 var decodeCollectionState = (state) => {
   invariant3(typeof state === "object" && state !== null, "Invalid state", {
     F: __dxlog_file4,
-    L: 557,
+    L: 592,
     S: void 0,
     A: [
       "typeof state === 'object' && state !== null",
@@ -1656,10 +1685,10 @@ var MeshEchoReplicator = class {
               documentId: params.documentId,
               peerId: connection.peerId
             });
-            log6("document not found locally for share policy check, accepting the remote document", {
+            log6("document not found locally for share policy check", {
               peerId: connection.peerId,
               documentId: params.documentId,
-              remoteDocumentExists
+              acceptDocument: remoteDocumentExists
             }, {
               F: __dxlog_file7,
               L: 91,
@@ -2923,7 +2952,7 @@ var INDEXER_CONFIG = {
   ]
 };
 var EchoHost = class extends Resource9 {
-  constructor({ kv, peerIdProvider }) {
+  constructor({ kv, peerIdProvider, getSpaceKeyByRootDocumentId }) {
     super();
     this._spaceStateManager = new SpaceStateManager();
     this._indexMetadataStore = new IndexMetadataStore({
@@ -2934,7 +2963,8 @@ var EchoHost = class extends Resource9 {
       db: kv,
       dataMonitor: this._echoDataMonitor,
       indexMetadataStore: this._indexMetadataStore,
-      peerIdProvider
+      peerIdProvider,
+      getSpaceKeyByRootDocumentId
     });
     this._indexer = new Indexer({
       db: kv,
@@ -3059,7 +3089,7 @@ var EchoHost = class extends Resource9 {
   async createSpaceRoot(spaceKey) {
     invariant11(this._lifecycleState === LifecycleState4.OPEN, void 0, {
       F: __dxlog_file15,
-      L: 217,
+      L: 220,
       S: this,
       A: [
         "this._lifecycleState === LifecycleState.OPEN",
@@ -3087,7 +3117,7 @@ var EchoHost = class extends Resource9 {
   async openSpaceRoot(spaceId, automergeUrl) {
     invariant11(this._lifecycleState === LifecycleState4.OPEN, void 0, {
       F: __dxlog_file15,
-      L: 236,
+      L: 239,
       S: this,
       A: [
         "this._lifecycleState === LifecycleState.OPEN",
@@ -3116,8 +3146,7 @@ var EchoHost = class extends Resource9 {
 };
 
 // packages/core/echo/echo-pipeline/src/edge/echo-edge-replicator.ts
-import { Mutex, scheduleTask as scheduleTask2, scheduleMicroTask } from "@dxos/async";
-import * as A5 from "@dxos/automerge/automerge";
+import { Mutex, scheduleTask as scheduleTask2, scheduleMicroTask, Trigger as Trigger2 } from "@dxos/async";
 import { cbor as cbor2 } from "@dxos/automerge/automerge-repo";
 import { Context as Context6, Resource as Resource10 } from "@dxos/context";
 import { randomUUID } from "@dxos/crypto";
@@ -3210,14 +3239,14 @@ var EchoEdgeReplicator = class {
       connectedSpaces: this._connectedSpaces.size
     }, {
       F: __dxlog_file16,
-      L: 60,
+      L: 59,
       S: this,
       C: (f, a) => f(...a)
     });
     this._context = context;
     this._ctx = Context6.default(void 0, {
       F: __dxlog_file16,
-      L: 63
+      L: 62
     });
     this._ctx.onDispose(this._edgeConnection.onReconnected(() => {
       this._ctx && scheduleMicroTask(this._ctx, () => this._handleReconnect());
@@ -3296,7 +3325,7 @@ var EchoEdgeReplicator = class {
   async _openConnection(spaceId, reconnects = 0) {
     invariant12(this._context, void 0, {
       F: __dxlog_file16,
-      L: 124,
+      L: 123,
       S: this,
       A: [
         "this._context",
@@ -3305,7 +3334,7 @@ var EchoEdgeReplicator = class {
     });
     invariant12(!this._connections.has(spaceId), void 0, {
       F: __dxlog_file16,
-      L: 125,
+      L: 124,
       S: this,
       A: [
         "!this._connections.has(spaceId)",
@@ -3335,7 +3364,7 @@ var EchoEdgeReplicator = class {
           restartDelay
         }, {
           F: __dxlog_file16,
-          L: 148,
+          L: 147,
           S: this,
           C: (f, a) => f(...a)
         });
@@ -3366,10 +3395,20 @@ var EchoEdgeReplicator = class {
     await connection.open();
   }
 };
+var MAX_INFLIGHT_REQUESTS = 5;
 var EdgeReplicatorConnection = class extends Resource10 {
   constructor({ edgeConnection, spaceId, context, sharedPolicyEnabled, onRemoteConnected, onRemoteDisconnected, onRestartRequested }) {
     super();
     this._remotePeerId = null;
+    /**
+    * Prevents sending too many messages to edge over this connection so that we don't overwhelm
+    * a replicator durable object.
+    * inflightRequests counter is incremented on outgoing sync messages and decremented on incoming messages.
+    * The trigger is waiting while the counter is above MAX_INFLIGHT_REQUESTS.
+    * The counter can go negative because we receive edge-initiated sync messages on doc change broadcasts.
+    */
+    this._outgoingRequestsBarrier = new Trigger2();
+    this._inflightRequests = 0;
     this._edgeConnection = edgeConnection;
     this._spaceId = spaceId;
     this._context = context;
@@ -3379,6 +3418,7 @@ var EdgeReplicatorConnection = class extends Resource10 {
     this._onRemoteConnected = onRemoteConnected;
     this._onRemoteDisconnected = onRemoteDisconnected;
     this._onRestartRequested = onRestartRequested;
+    this._outgoingRequestsBarrier.wake();
     this.readable = new ReadableStream({
       start: (controller) => {
         this._readableStreamController = controller;
@@ -3386,6 +3426,11 @@ var EdgeReplicatorConnection = class extends Resource10 {
     });
     this.writable = new WritableStream({
       write: async (message, controller) => {
+        await this._outgoingRequestsBarrier.wait();
+        this._inflightRequests++;
+        if (this._inflightRequests === MAX_INFLIGHT_REQUESTS) {
+          this._outgoingRequestsBarrier.reset();
+        }
         await this._sendMessage(message);
       }
     });
@@ -3393,7 +3438,7 @@ var EdgeReplicatorConnection = class extends Resource10 {
   async _open(ctx) {
     log11("open", void 0, {
       F: __dxlog_file16,
-      L: 242,
+      L: 261,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -3405,17 +3450,18 @@ var EdgeReplicatorConnection = class extends Resource10 {
   async _close() {
     log11("close", void 0, {
       F: __dxlog_file16,
-      L: 254,
+      L: 273,
       S: this,
       C: (f, a) => f(...a)
     });
     this._readableStreamController.close();
+    this._outgoingRequestsBarrier.throw(new Error("Connection closed."));
     await this._onRemoteDisconnected();
   }
   get peerId() {
     invariant12(this._remotePeerId, "Not connected", {
       F: __dxlog_file16,
-      L: 260,
+      L: 282,
       S: this,
       A: [
         "this._remotePeerId",
@@ -3434,12 +3480,13 @@ var EdgeReplicatorConnection = class extends Resource10 {
         documentId: params.documentId,
         peerId: this._remotePeerId
       });
-      log11.info("document not found locally for share policy check, accepting the remote document", {
+      log11.verbose("edge-replicator document not found locally for share policy check", {
         documentId: params.documentId,
-        remoteDocumentExists
+        acceptDocument: remoteDocumentExists,
+        remoteId: this._remotePeerId
       }, {
         F: __dxlog_file16,
-        L: 275,
+        L: 297,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -3452,23 +3499,20 @@ var EdgeReplicatorConnection = class extends Resource10 {
       return true;
     }
     const spaceId = getSpaceIdFromCollectionId(params.collectionId);
-    return spaceId === this._spaceId;
+    return spaceId === this._spaceId && params.collectionId.split(":").length === 3;
   }
   _onMessage(message) {
     if (message.serviceId !== this._targetServiceId) {
       return;
     }
     const payload = cbor2.decode(message.payload.value);
-    log11("recv", () => {
-      const decodedData = payload.type === "sync" && payload.data ? A5.decodeSyncMessage(payload.data) : payload.type === "collection-state" ? payload.state : payload;
-      return {
-        from: message.serviceId,
-        type: payload.type,
-        decodedData
-      };
+    log11.verbose("edge replicator receive", {
+      type: payload.type,
+      documentId: payload.type === "sync" && payload.documentId,
+      remoteId: this._remotePeerId
     }, {
       F: __dxlog_file16,
-      L: 302,
+      L: 326,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -3480,18 +3524,23 @@ var EdgeReplicatorConnection = class extends Resource10 {
       this._onRestartRequested();
       return;
     }
+    if (message.type === "sync") {
+      this._inflightRequests--;
+      if (this._inflightRequests === MAX_INFLIGHT_REQUESTS - 1) {
+        this._outgoingRequestsBarrier.wake();
+      }
+    }
     this._readableStreamController.enqueue(message);
   }
   async _sendMessage(message) {
     message.targetId = this._targetServiceId;
-    log11("send", {
+    log11.verbose("edge replicator send", {
       type: message.type,
-      senderId: message.senderId,
-      targetId: message.targetId,
-      documentId: message.documentId
+      documentId: message.type === "sync" && message.documentId,
+      remoteId: this._remotePeerId
     }, {
       F: __dxlog_file16,
-      L: 332,
+      L: 360,
       S: this,
       C: (f, a) => f(...a)
     });