@dxos/echo-pipeline 0.7.4-staging.f7e8224 → 0.7.5-main.499c70c

package/dist/lib/browser/index.mjs

@@ -223,6 +223,7 @@ var CollectionSynchronizer = class extends Resource2 {
     * CollectionId -> State.
     */
     this._perCollectionStates = /* @__PURE__ */ new Map();
+    this._activeCollections = /* @__PURE__ */ new Set();
     this._connectedPeers = /* @__PURE__ */ new Set();
     this.remoteStateUpdated = new Event();
     this._sendCollectionState = params.sendCollectionState;
@@ -232,43 +233,58 @@ var CollectionSynchronizer = class extends Resource2 {
   async _open(ctx) {
     scheduleTaskInterval(this._ctx, async () => {
       for (const collectionId of this._perCollectionStates.keys()) {
-        this.refreshCollection(collectionId);
-        await asyncReturn();
+        if (this._activeCollections.has(collectionId)) {
+          this.refreshCollection(collectionId);
+          await asyncReturn();
+        }
       }
     }, POLL_INTERVAL);
   }
   getRegisteredCollectionIds() {
     return [
-      ...this._perCollectionStates.keys()
+      ...this._activeCollections
     ];
   }
   getLocalCollectionState(collectionId) {
-    return this._getPerCollectionState(collectionId).localState;
+    return this._perCollectionStates.get(collectionId)?.localState;
   }
   setLocalCollectionState(collectionId, state) {
+    this._activeCollections.add(collectionId);
     log2("setLocalCollectionState", {
       collectionId,
       state
     }, {
       F: __dxlog_file2,
-      L: 68,
+      L: 73,
       S: this,
       C: (f, a) => f(...a)
     });
-    this._getPerCollectionState(collectionId).localState = state;
+    this._getOrCreatePerCollectionState(collectionId).localState = state;
     queueMicrotask(async () => {
-      if (!this._ctx.disposed) {
+      if (!this._ctx.disposed && this._activeCollections.has(collectionId)) {
         this._refreshInterestedPeers(collectionId);
         this.refreshCollection(collectionId);
       }
     });
   }
+  clearLocalCollectionState(collectionId) {
+    this._activeCollections.delete(collectionId);
+    this._perCollectionStates.delete(collectionId);
+    log2("clearLocalCollectionState", {
+      collectionId
+    }, {
+      F: __dxlog_file2,
+      L: 87,
+      S: this,
+      C: (f, a) => f(...a)
+    });
+  }
   getRemoteCollectionStates(collectionId) {
-    return this._getPerCollectionState(collectionId).remoteStates;
+    return this._getOrCreatePerCollectionState(collectionId).remoteStates;
   }
   refreshCollection(collectionId) {
     let scheduleAnotherRefresh = false;
-    const state = this._getPerCollectionState(collectionId);
+    const state = this._getOrCreatePerCollectionState(collectionId);
     for (const peerId of this._connectedPeers) {
       if (state.interestedPeers.has(peerId)) {
         const lastQueried = state.lastQueried.get(peerId) ?? 0;
@@ -294,7 +310,7 @@ var CollectionSynchronizer = class extends Resource2 {
         return;
       }
       for (const [collectionId, state] of this._perCollectionStates.entries()) {
-        if (this._shouldSyncCollection(collectionId, peerId)) {
+        if (this._activeCollections.has(collectionId) && this._shouldSyncCollection(collectionId, peerId)) {
           state.interestedPeers.add(peerId);
           state.lastQueried.set(peerId, Date.now());
           this._queryCollectionState(collectionId, peerId);
@@ -315,7 +331,7 @@ var CollectionSynchronizer = class extends Resource2 {
   * Callback when a peer queries the state of a collection.
   */
   onCollectionStateQueried(collectionId, peerId) {
-    const perCollectionState = this._getPerCollectionState(collectionId);
+    const perCollectionState = this._getOrCreatePerCollectionState(collectionId);
     if (perCollectionState.localState) {
       this._sendCollectionState(collectionId, peerId, perCollectionState.localState);
     }
@@ -330,19 +346,19 @@ var CollectionSynchronizer = class extends Resource2 {
       state
     }, {
       F: __dxlog_file2,
-      L: 148,
+      L: 159,
       S: this,
       C: (f, a) => f(...a)
     });
     validateCollectionState(state);
-    const perCollectionState = this._getPerCollectionState(collectionId);
+    const perCollectionState = this._getOrCreatePerCollectionState(collectionId);
     perCollectionState.remoteStates.set(peerId, state);
     this.remoteStateUpdated.emit({
       peerId,
       collectionId
     });
   }
-  _getPerCollectionState(collectionId) {
+  _getOrCreatePerCollectionState(collectionId) {
     return defaultMap(this._perCollectionStates, collectionId, () => ({
       localState: void 0,
       remoteStates: /* @__PURE__ */ new Map(),
@@ -353,9 +369,9 @@ var CollectionSynchronizer = class extends Resource2 {
   _refreshInterestedPeers(collectionId) {
     for (const peerId of this._connectedPeers) {
       if (this._shouldSyncCollection(collectionId, peerId)) {
-        this._getPerCollectionState(collectionId).interestedPeers.add(peerId);
+        this._getOrCreatePerCollectionState(collectionId).interestedPeers.add(peerId);
       } else {
-        this._getPerCollectionState(collectionId).interestedPeers.delete(peerId);
+        this._getOrCreatePerCollectionState(collectionId).interestedPeers.delete(peerId);
       }
     }
   }
@@ -1241,6 +1257,9 @@ var AutomergeHost = class extends Resource4 {
       documents
     });
   }
+  async clearLocalCollectionState(collectionId) {
+    this._collectionSynchronizer.clearLocalCollectionState(collectionId);
+  }
   _onCollectionStateQueried(collectionId, peerId) {
     this._collectionSynchronizer.onCollectionStateQueried(collectionId, peerId);
   }
@@ -1274,11 +1293,14 @@ var AutomergeHost = class extends Resource4 {
     if (toReplicate.length === 0) {
       return;
     }
-    log4.info("replication documents after collection sync", {
+    log4.info("replicating documents after collection sync", {
+      collectionId,
+      peerId,
+      toReplicate,
       count: toReplicate.length
     }, {
       F: __dxlog_file4,
-      L: 495,
+      L: 499,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -1345,7 +1367,7 @@ var changeIsPresentInDoc = (doc, changeHash) => {
 var decodeCollectionState = (state) => {
   invariant3(typeof state === "object" && state !== null, "Invalid state", {
     F: __dxlog_file4,
-    L: 553,
+    L: 560,
     S: void 0,
     A: [
       "typeof state === 'object' && state !== null",
@@ -1514,12 +1536,12 @@ var logSendSync = (message) => {
 import { invariant as invariant5 } from "@dxos/invariant";
 import { SpaceId } from "@dxos/keys";
 var __dxlog_file6 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/space-collection.ts";
-var deriveCollectionIdFromSpaceId = (spaceId) => `space:${spaceId}`;
+var deriveCollectionIdFromSpaceId = (spaceId, rootDocumentId) => rootDocumentId ? `space:${spaceId}:${rootDocumentId}` : `space:${spaceId}`;
 var getSpaceIdFromCollectionId = (collectionId) => {
-  const spaceId = collectionId.replace(/^space:/, "");
+  const spaceId = collectionId.split(":")[1];
   invariant5(SpaceId.isValid(spaceId), void 0, {
     F: __dxlog_file6,
-    L: 13,
+    L: 15,
     S: void 0,
     A: [
       "SpaceId.isValid(spaceId)",
@@ -1637,10 +1659,10 @@ var MeshEchoReplicator = class {
               documentId: params.documentId,
               peerId: connection.peerId
             });
-            log6("document not found locally for share policy check, accepting the remote document", {
+            log6("document not found locally for share policy check", {
               peerId: connection.peerId,
               documentId: params.documentId,
-              remoteDocumentExists
+              acceptDocument: remoteDocumentExists
             }, {
               F: __dxlog_file7,
               L: 91,
@@ -2103,6 +2125,7 @@ var DataServiceImpl = class {
     */
     this._subscriptions = /* @__PURE__ */ new Map();
     this._automergeHost = params.automergeHost;
+    this._spaceStateManager = params.spaceStateManager;
     this._updateIndexes = params.updateIndexes;
   }
   subscribe(request) {
@@ -2116,7 +2139,7 @@ var DataServiceImpl = class {
         ready();
       }).catch((err) => log7.catch(err, void 0, {
         F: __dxlog_file8,
-        L: 66,
+        L: 70,
         S: this,
         C: (f, a) => f(...a)
       }));
@@ -2127,7 +2150,7 @@ var DataServiceImpl = class {
     const synchronizer = this._subscriptions.get(request.subscriptionId);
     invariant7(synchronizer, "Subscription not found", {
       F: __dxlog_file8,
-      L: 73,
+      L: 77,
       S: this,
       A: [
         "synchronizer",
@@ -2148,7 +2171,7 @@ var DataServiceImpl = class {
     const synchronizer = this._subscriptions.get(request.subscriptionId);
     invariant7(synchronizer, "Subscription not found", {
       F: __dxlog_file8,
-      L: 88,
+      L: 92,
       S: this,
       A: [
         "synchronizer",
@@ -2190,18 +2213,29 @@ var DataServiceImpl = class {
   }
   subscribeSpaceSyncState(request) {
     return new Stream(({ ctx, next, ready }) => {
-      invariant7(SpaceId2.isValid(request.spaceId), void 0, {
+      const spaceId = request.spaceId;
+      invariant7(SpaceId2.isValid(spaceId), void 0, {
         F: __dxlog_file8,
-        L: 127,
+        L: 132,
         S: this,
         A: [
-          "SpaceId.isValid(request.spaceId)",
+          "SpaceId.isValid(spaceId)",
           ""
         ]
       });
-      const collectionId = deriveCollectionIdFromSpaceId(request.spaceId);
+      const rootDocumentId = this._spaceStateManager.getSpaceRootDocumentId(spaceId);
+      let collectionId = rootDocumentId && deriveCollectionIdFromSpaceId(spaceId, rootDocumentId);
+      this._spaceStateManager.spaceDocumentListUpdated.on(ctx, (event) => {
+        const newId = deriveCollectionIdFromSpaceId(spaceId, event.spaceRootId);
+        if (newId !== collectionId) {
+          collectionId = newId;
+          scheduler.trigger();
+        }
+      });
       const scheduler = new UpdateScheduler2(ctx, async () => {
-        const state = await this._automergeHost.getCollectionSyncState(collectionId);
+        const state = collectionId ? await this._automergeHost.getCollectionSyncState(collectionId) : {
+          peers: []
+        };
         next({
           peers: state.peers.map((peer) => ({
             peerId: peer.peerId,
@@ -2827,6 +2861,9 @@ var SpaceStateManager = class extends Resource8 {
   getRootByDocumentId(documentId) {
     return this._roots.get(documentId);
   }
+  getSpaceRootDocumentId(spaceId) {
+    return this._rootBySpace.get(spaceId);
+  }
   async assignRootToSpace(spaceId, handle) {
     let root;
     if (this._roots.has(handle.documentId)) {
@@ -2846,7 +2883,7 @@ var SpaceStateManager = class extends Resource8 {
     this._rootBySpace.set(spaceId, root.handle.documentId);
     const ctx = new Context5(void 0, {
       F: __dxlog_file14,
-      L: 58
+      L: 62
     });
     this._perRootContext.set(root.handle.documentId, ctx);
     await root.handle.whenReady();
@@ -2857,7 +2894,7 @@ var SpaceStateManager = class extends Resource8 {
       ];
       if (!isEqual(documentIds, this._lastSpaceDocumentList.get(spaceId))) {
         this._lastSpaceDocumentList.set(spaceId, documentIds);
-        this.spaceDocumentListUpdated.emit(new SpaceDocumentListUpdatedEvent(spaceId, documentIds));
+        this.spaceDocumentListUpdated.emit(new SpaceDocumentListUpdatedEvent(spaceId, root.documentId, prevRootId, documentIds));
       }
     }, {
       maxFrequency: 50
@@ -2870,8 +2907,10 @@ var SpaceStateManager = class extends Resource8 {
   }
 };
 var SpaceDocumentListUpdatedEvent = class {
-  constructor(spaceId, documentIds) {
+  constructor(spaceId, spaceRootId, previousRootId, documentIds) {
     this.spaceId = spaceId;
+    this.spaceRootId = spaceRootId;
+    this.previousRootId = previousRootId;
     this.documentIds = documentIds;
   }
 };
@@ -2916,6 +2955,7 @@ var EchoHost = class extends Resource9 {
     });
     this._dataService = new DataServiceImpl({
       automergeHost: this._automergeHost,
+      spaceStateManager: this._spaceStateManager,
       updateIndexes: async () => {
         await this._indexer.updateIndexes();
       }
@@ -2979,7 +3019,11 @@ var EchoHost = class extends Resource9 {
     await this._queryService.open(ctx);
     await this._spaceStateManager.open(ctx);
     this._spaceStateManager.spaceDocumentListUpdated.on(this._ctx, (e) => {
+      if (e.previousRootId) {
+        void this._automergeHost.clearLocalCollectionState(deriveCollectionIdFromSpaceId(e.spaceId, e.previousRootId));
+      }
       void this._automergeHost.updateLocalCollectionState(deriveCollectionIdFromSpaceId(e.spaceId), e.documentIds);
+      void this._automergeHost.updateLocalCollectionState(deriveCollectionIdFromSpaceId(e.spaceId, e.spaceRootId), e.documentIds);
     });
   }
   async _close(ctx) {
@@ -3018,7 +3062,7 @@ var EchoHost = class extends Resource9 {
   async createSpaceRoot(spaceKey) {
     invariant11(this._lifecycleState === LifecycleState4.OPEN, void 0, {
       F: __dxlog_file15,
-      L: 209,
+      L: 217,
       S: this,
       A: [
         "this._lifecycleState === LifecycleState.OPEN",
@@ -3046,7 +3090,7 @@ var EchoHost = class extends Resource9 {
   async openSpaceRoot(spaceId, automergeUrl) {
     invariant11(this._lifecycleState === LifecycleState4.OPEN, void 0, {
       F: __dxlog_file15,
-      L: 228,
+      L: 236,
       S: this,
       A: [
         "this._lifecycleState === LifecycleState.OPEN",
@@ -3072,15 +3116,10 @@ var EchoHost = class extends Resource9 {
   async removeReplicator(replicator) {
     await this._automergeHost.removeReplicator(replicator);
   }
-  async getSpaceSyncState(spaceId) {
-    const collectionId = deriveCollectionIdFromSpaceId(spaceId);
-    return this._automergeHost.getCollectionSyncState(collectionId);
-  }
 };
 
 // packages/core/echo/echo-pipeline/src/edge/echo-edge-replicator.ts
-import { Mutex, scheduleTask as scheduleTask2, scheduleMicroTask } from "@dxos/async";
-import * as A5 from "@dxos/automerge/automerge";
+import { Mutex, scheduleTask as scheduleTask2, scheduleMicroTask, Trigger as Trigger2 } from "@dxos/async";
 import { cbor as cbor2 } from "@dxos/automerge/automerge-repo";
 import { Context as Context6, Resource as Resource10 } from "@dxos/context";
 import { randomUUID } from "@dxos/crypto";
@@ -3173,14 +3212,14 @@ var EchoEdgeReplicator = class {
       connectedSpaces: this._connectedSpaces.size
     }, {
       F: __dxlog_file16,
-      L: 60,
+      L: 59,
       S: this,
       C: (f, a) => f(...a)
     });
     this._context = context;
     this._ctx = Context6.default(void 0, {
       F: __dxlog_file16,
-      L: 63
+      L: 62
     });
     this._ctx.onDispose(this._edgeConnection.onReconnected(() => {
       this._ctx && scheduleMicroTask(this._ctx, () => this._handleReconnect());
@@ -3259,7 +3298,7 @@ var EchoEdgeReplicator = class {
   async _openConnection(spaceId, reconnects = 0) {
     invariant12(this._context, void 0, {
       F: __dxlog_file16,
-      L: 124,
+      L: 123,
       S: this,
       A: [
         "this._context",
@@ -3268,7 +3307,7 @@ var EchoEdgeReplicator = class {
     });
     invariant12(!this._connections.has(spaceId), void 0, {
       F: __dxlog_file16,
-      L: 125,
+      L: 124,
       S: this,
       A: [
         "!this._connections.has(spaceId)",
@@ -3298,7 +3337,7 @@ var EchoEdgeReplicator = class {
           restartDelay
         }, {
           F: __dxlog_file16,
-          L: 148,
+          L: 147,
           S: this,
           C: (f, a) => f(...a)
         });
@@ -3329,10 +3368,20 @@ var EchoEdgeReplicator = class {
     await connection.open();
   }
 };
+var MAX_INFLIGHT_REQUESTS = 5;
 var EdgeReplicatorConnection = class extends Resource10 {
   constructor({ edgeConnection, spaceId, context, sharedPolicyEnabled, onRemoteConnected, onRemoteDisconnected, onRestartRequested }) {
     super();
     this._remotePeerId = null;
+    /**
+    * Prevents sending too many messages to edge over this connection so that we don't overwhelm
+    * a replicator durable object.
+    * inflightRequests counter is incremented on outgoing sync messages and decremented on incoming messages.
+    * The trigger is waiting while the counter is above MAX_INFLIGHT_REQUESTS.
+    * The counter can go negative because we receive edge-initiated sync messages on doc change broadcasts.
+    */
+    this._outgoingRequestsBarrier = new Trigger2();
+    this._inflightRequests = 0;
     this._edgeConnection = edgeConnection;
     this._spaceId = spaceId;
     this._context = context;
@@ -3342,6 +3391,7 @@ var EdgeReplicatorConnection = class extends Resource10 {
     this._onRemoteConnected = onRemoteConnected;
     this._onRemoteDisconnected = onRemoteDisconnected;
     this._onRestartRequested = onRestartRequested;
+    this._outgoingRequestsBarrier.wake();
     this.readable = new ReadableStream({
       start: (controller) => {
         this._readableStreamController = controller;
@@ -3349,6 +3399,11 @@ var EdgeReplicatorConnection = class extends Resource10 {
     });
     this.writable = new WritableStream({
       write: async (message, controller) => {
+        await this._outgoingRequestsBarrier.wait();
+        this._inflightRequests++;
+        if (this._inflightRequests === MAX_INFLIGHT_REQUESTS) {
+          this._outgoingRequestsBarrier.reset();
+        }
         await this._sendMessage(message);
       }
     });
@@ -3356,7 +3411,7 @@ var EdgeReplicatorConnection = class extends Resource10 {
   async _open(ctx) {
     log11("open", void 0, {
       F: __dxlog_file16,
-      L: 242,
+      L: 261,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -3368,17 +3423,18 @@ var EdgeReplicatorConnection = class extends Resource10 {
   async _close() {
     log11("close", void 0, {
       F: __dxlog_file16,
-      L: 254,
+      L: 273,
       S: this,
       C: (f, a) => f(...a)
     });
     this._readableStreamController.close();
+    this._outgoingRequestsBarrier.throw(new Error("Connection closed."));
     await this._onRemoteDisconnected();
   }
   get peerId() {
     invariant12(this._remotePeerId, "Not connected", {
       F: __dxlog_file16,
-      L: 260,
+      L: 282,
       S: this,
       A: [
         "this._remotePeerId",
@@ -3397,12 +3453,13 @@ var EdgeReplicatorConnection = class extends Resource10 {
         documentId: params.documentId,
         peerId: this._remotePeerId
       });
-      log11.info("document not found locally for share policy check, accepting the remote document", {
+      log11.verbose("edge-replicator document not found locally for share policy check", {
         documentId: params.documentId,
-        remoteDocumentExists
+        acceptDocument: remoteDocumentExists,
+        remoteId: this._remotePeerId
       }, {
         F: __dxlog_file16,
-        L: 275,
+        L: 297,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -3415,23 +3472,20 @@ var EdgeReplicatorConnection = class extends Resource10 {
       return true;
     }
     const spaceId = getSpaceIdFromCollectionId(params.collectionId);
-    return spaceId === this._spaceId;
+    return spaceId === this._spaceId && params.collectionId.split(":").length === 3;
   }
   _onMessage(message) {
     if (message.serviceId !== this._targetServiceId) {
       return;
     }
     const payload = cbor2.decode(message.payload.value);
-    log11("recv", () => {
-      const decodedData = payload.type === "sync" && payload.data ? A5.decodeSyncMessage(payload.data) : payload.type === "collection-state" ? payload.state : payload;
-      return {
-        from: message.serviceId,
-        type: payload.type,
-        decodedData
-      };
+    log11.verbose("edge replicator receive", {
+      type: payload.type,
+      documentId: payload.type === "sync" && payload.documentId,
+      remoteId: this._remotePeerId
     }, {
       F: __dxlog_file16,
-      L: 302,
+      L: 326,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -3443,18 +3497,23 @@ var EdgeReplicatorConnection = class extends Resource10 {
       this._onRestartRequested();
       return;
     }
+    if (message.type === "sync") {
+      this._inflightRequests--;
+      if (this._inflightRequests === MAX_INFLIGHT_REQUESTS - 1) {
+        this._outgoingRequestsBarrier.wake();
+      }
+    }
     this._readableStreamController.enqueue(message);
   }
   async _sendMessage(message) {
     message.targetId = this._targetServiceId;
-    log11("send", {
+    log11.verbose("edge replicator send", {
       type: message.type,
-      senderId: message.senderId,
-      targetId: message.targetId,
-      documentId: message.documentId
+      documentId: message.type === "sync" && message.documentId,
+      remoteId: this._remotePeerId
     }, {
       F: __dxlog_file16,
-      L: 332,
+      L: 360,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -3508,9 +3567,11 @@ export {
   QueryServiceImpl,
   QueryState,
   Space,
+  SpaceDocumentListUpdatedEvent,
   SpaceManager,
   SpaceProtocol,
   SpaceProtocolSession,
+  SpaceStateManager,
   TimeframeClock,
   codec,
   createIdFromSpaceKey,