@dxos/echo-pipeline 0.7.4 → 0.7.5-main.9cb18ac

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dxos/echo-pipeline",
-  "version": "0.7.4",
+  "version": "0.7.5-main.9cb18ac",
   "description": "ECHO database.",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
@@ -37,44 +37,44 @@
     "crc-32": "^1.2.2",
     "level-transcoder": "^1.0.1",
     "lodash.isequal": "^4.5.0",
-    "@dxos/async": "0.7.4",
-    "@dxos/automerge": "0.7.4",
-    "@dxos/context": "0.7.4",
-    "@dxos/codec-protobuf": "0.7.4",
-    "@dxos/credentials": "0.7.4",
-    "@dxos/crypto": "0.7.4",
-    "@dxos/debug": "0.7.4",
-    "@dxos/feed-store": "0.7.4",
-    "@dxos/hypercore": "0.7.4",
-    "@dxos/echo-protocol": "0.7.4",
-    "@dxos/echo-schema": "0.7.4",
-    "@dxos/edge-client": "0.7.4",
-    "@dxos/invariant": "0.7.4",
-    "@dxos/indexing": "0.7.4",
-    "@dxos/keyring": "0.7.4",
-    "@dxos/kv-store": "0.7.4",
-    "@dxos/log": "0.7.4",
-    "@dxos/keys": "0.7.4",
-    "@dxos/messaging": "0.7.4",
-    "@dxos/node-std": "0.7.4",
-    "@dxos/protocols": "0.7.4",
-    "@dxos/random-access-storage": "0.7.4",
-    "@dxos/network-manager": "0.7.4",
-    "@dxos/teleport": "0.7.4",
-    "@dxos/teleport-extension-automerge-replicator": "0.7.4",
-    "@dxos/teleport-extension-object-sync": "0.7.4",
-    "@dxos/teleport-extension-gossip": "0.7.4",
-    "@dxos/timeframe": "0.7.4",
-    "@dxos/teleport-extension-replicator": "0.7.4",
-    "@dxos/typings": "0.7.4",
-    "@dxos/tracing": "0.7.4",
-    "@dxos/util": "0.7.4"
+    "@dxos/async": "0.7.5-main.9cb18ac",
+    "@dxos/codec-protobuf": "0.7.5-main.9cb18ac",
+    "@dxos/automerge": "0.7.5-main.9cb18ac",
+    "@dxos/context": "0.7.5-main.9cb18ac",
+    "@dxos/credentials": "0.7.5-main.9cb18ac",
+    "@dxos/crypto": "0.7.5-main.9cb18ac",
+    "@dxos/debug": "0.7.5-main.9cb18ac",
+    "@dxos/echo-protocol": "0.7.5-main.9cb18ac",
+    "@dxos/echo-schema": "0.7.5-main.9cb18ac",
+    "@dxos/edge-client": "0.7.5-main.9cb18ac",
+    "@dxos/feed-store": "0.7.5-main.9cb18ac",
+    "@dxos/indexing": "0.7.5-main.9cb18ac",
+    "@dxos/invariant": "0.7.5-main.9cb18ac",
+    "@dxos/hypercore": "0.7.5-main.9cb18ac",
+    "@dxos/keyring": "0.7.5-main.9cb18ac",
+    "@dxos/keys": "0.7.5-main.9cb18ac",
+    "@dxos/log": "0.7.5-main.9cb18ac",
+    "@dxos/messaging": "0.7.5-main.9cb18ac",
+    "@dxos/node-std": "0.7.5-main.9cb18ac",
+    "@dxos/network-manager": "0.7.5-main.9cb18ac",
+    "@dxos/kv-store": "0.7.5-main.9cb18ac",
+    "@dxos/protocols": "0.7.5-main.9cb18ac",
+    "@dxos/teleport-extension-automerge-replicator": "0.7.5-main.9cb18ac",
+    "@dxos/teleport-extension-gossip": "0.7.5-main.9cb18ac",
+    "@dxos/random-access-storage": "0.7.5-main.9cb18ac",
+    "@dxos/teleport": "0.7.5-main.9cb18ac",
+    "@dxos/teleport-extension-object-sync": "0.7.5-main.9cb18ac",
+    "@dxos/teleport-extension-replicator": "0.7.5-main.9cb18ac",
+    "@dxos/tracing": "0.7.5-main.9cb18ac",
+    "@dxos/typings": "0.7.5-main.9cb18ac",
+    "@dxos/timeframe": "0.7.5-main.9cb18ac",
+    "@dxos/util": "0.7.5-main.9cb18ac"
   },
   "devDependencies": {
     "@types/lodash.isequal": "^4.5.0",
     "fast-check": "^3.19.0",
     "get-port-please": "^3.1.1",
-    "@dxos/test-utils": "0.7.4"
+    "@dxos/test-utils": "0.7.5-main.9cb18ac"
   },
   "publishConfig": {
     "access": "public"

package/src/automerge/automerge-host.ts

@@ -44,6 +44,8 @@ import { LevelDBStorageAdapter, type BeforeSaveParams } from './leveldb-storage-
 
 export type PeerIdProvider = () => string | undefined;
 
+export type RootDocumentSpaceKeyProvider = (documentId: string) => PublicKey | undefined;
+
 export type AutomergeHostParams = {
   db: LevelDB;
 
@@ -54,6 +56,7 @@ export type AutomergeHostParams = {
    * Used for creating stable ids. A random key is generated on open, if no value is provided.
    */
   peerIdProvider?: PeerIdProvider;
+  getSpaceKeyByRootDocumentId?: RootDocumentSpaceKeyProvider;
 };
 
 export type LoadDocOptions = {
@@ -90,10 +93,17 @@ export class AutomergeHost extends Resource {
   private _peerId!: PeerId;
 
   private readonly _peerIdProvider?: PeerIdProvider;
+  private readonly _getSpaceKeyByRootDocumentId?: RootDocumentSpaceKeyProvider;
 
   public readonly collectionStateUpdated = new Event<{ collectionId: CollectionId }>();
 
-  constructor({ db, indexMetadataStore, dataMonitor, peerIdProvider }: AutomergeHostParams) {
+  constructor({
+    db,
+    indexMetadataStore,
+    dataMonitor,
+    peerIdProvider,
+    getSpaceKeyByRootDocumentId,
+  }: AutomergeHostParams) {
     super();
     this._db = db;
     this._storage = new LevelDBStorageAdapter({
@@ -114,6 +124,7 @@ export class AutomergeHost extends Resource {
     this._headsStore = new HeadsStore({ db: db.sublevel('heads') });
     this._indexMetadataStore = indexMetadataStore;
     this._peerIdProvider = peerIdProvider;
+    this._getSpaceKeyByRootDocumentId = getSpaceKeyByRootDocumentId;
   }
 
   protected override async _open() {
@@ -132,14 +143,28 @@ export class AutomergeHost extends Resource {
       ],
     });
 
-    Event.wrap(this._echoNetworkAdapter, 'peer-candidate').on(this._ctx, ((e: PeerCandidatePayload) =>
-      this._onPeerConnected(e.peerId)) as any);
-    Event.wrap(this._echoNetworkAdapter, 'peer-disconnected').on(this._ctx, ((e: PeerDisconnectedPayload) =>
-      this._onPeerDisconnected(e.peerId)) as any);
+    let updatingAuthScope = false;
+    Event.wrap(this._echoNetworkAdapter, 'peer-candidate').on(
+      this._ctx,
+      ((e: PeerCandidatePayload) => !updatingAuthScope && this._onPeerConnected(e.peerId)) as any,
+    );
+    Event.wrap(this._echoNetworkAdapter, 'peer-disconnected').on(
+      this._ctx,
+      ((e: PeerDisconnectedPayload) => !updatingAuthScope && this._onPeerDisconnected(e.peerId)) as any,
+    );
 
-    this._collectionSynchronizer.remoteStateUpdated.on(this._ctx, ({ collectionId, peerId }) => {
+    this._collectionSynchronizer.remoteStateUpdated.on(this._ctx, ({ collectionId, peerId, newDocsAppeared }) => {
       this._onRemoteCollectionStateUpdated(collectionId, peerId);
       this.collectionStateUpdated.emit({ collectionId: collectionId as CollectionId });
+      // We use collection lookups during share policy check, so we might need to update share policy for the new doc
+      if (newDocsAppeared) {
+        updatingAuthScope = true;
+        try {
+          this._echoNetworkAdapter.onConnectionAuthScopeChanged(peerId);
+        } finally {
+          updatingAuthScope = false;
+        }
+      }
     });
 
     await this._echoNetworkAdapter.open();
@@ -351,16 +376,23 @@ export class AutomergeHost extends Resource {
 
   private async _getContainingSpaceForDocument(documentId: string): Promise<PublicKey | null> {
     const doc = this._repo.handles[documentId as any]?.docSync();
-    if (!doc) {
-      return null;
+    if (doc) {
+      const spaceKeyHex = getSpaceKeyFromDoc(doc);
+      if (spaceKeyHex) {
+        return PublicKey.from(spaceKeyHex);
+      }
     }
-
-    const spaceKeyHex = getSpaceKeyFromDoc(doc);
-    if (!spaceKeyHex) {
-      return null;
+    /**
+     * Edge case on the initial space setup.
+     * A peer is maybe trying to share space root document with us after a successful invitation.
+     * We don't have a document to check access block locally, so we need to rely on external sources (space metada).
+     */
+    const rootDocSpaceKey = this._getSpaceKeyByRootDocumentId?.(documentId);
+    if (rootDocSpaceKey) {
+      return rootDocSpaceKey;
     }
 
-    return PublicKey.from(spaceKeyHex);
+    return null;
   }
 
   /**
@@ -496,7 +528,10 @@ export class AutomergeHost extends Resource {
       return;
     }
 
-    log.info('replication documents after collection sync', {
+    log.info('replicating documents after collection sync', {
+      collectionId,
+      peerId,
+      toReplicate,
       count: toReplicate.length,
     });
 

package/src/automerge/collection-synchronizer.ts

@@ -35,7 +35,7 @@ export class CollectionSynchronizer extends Resource {
 
   private readonly _connectedPeers = new Set<PeerId>();
 
-  public readonly remoteStateUpdated = new Event<{ collectionId: string; peerId: PeerId }>();
+  public readonly remoteStateUpdated = new Event<{ collectionId: string; peerId: PeerId; newDocsAppeared: boolean }>();
 
   constructor(params: CollectionSynchronizerParams) {
     super();
@@ -121,7 +121,7 @@ export class CollectionSynchronizer extends Resource {
         return;
       }
       for (const [collectionId, state] of this._perCollectionStates.entries()) {
-        if (this._shouldSyncCollection(collectionId, peerId)) {
+        if (this._activeCollections.has(collectionId) && this._shouldSyncCollection(collectionId, peerId)) {
           state.interestedPeers.add(peerId);
           state.lastQueried.set(peerId, Date.now());
           this._queryCollectionState(collectionId, peerId);
@@ -159,8 +159,12 @@ export class CollectionSynchronizer extends Resource {
     log('onRemoteStateReceived', { collectionId, peerId, state });
     validateCollectionState(state);
     const perCollectionState = this._getOrCreatePerCollectionState(collectionId);
-    perCollectionState.remoteStates.set(peerId, state);
-    this.remoteStateUpdated.emit({ peerId, collectionId });
+    const existingState = perCollectionState.remoteStates.get(peerId) ?? { documents: {} };
+    const diff = diffCollectionState(existingState, state);
+    if (diff.missingOnLocal.length > 0 || diff.different.length > 0) {
+      perCollectionState.remoteStates.set(peerId, state);
+      this.remoteStateUpdated.emit({ peerId, collectionId, newDocsAppeared: diff.missingOnLocal.length > 0 });
+    }
   }
 
   private _getOrCreatePerCollectionState(collectionId: string): PerCollectionState {

package/src/automerge/echo-network-adapter.ts

@@ -246,6 +246,13 @@ export class EchoNetworkAdapter extends NetworkAdapter {
     this._params.monitor?.recordMessageReceived(message);
   }
 
+  public onConnectionAuthScopeChanged(peer: PeerId) {
+    const entry = this._connections.get(peer);
+    if (entry) {
+      this._onConnectionAuthScopeChanged(entry.connection);
+    }
+  }
+
   /**
    * Trigger doc-synchronizer shared documents set recalculation. Happens on peer-candidate.
    * TODO(y): replace with a proper API call when sharePolicy update becomes supported by automerge-repo

package/src/automerge/mesh-echo-replicator.ts

@@ -88,10 +88,10 @@ export class MeshEchoReplicator implements EchoReplicator {
               documentId: params.documentId,
               peerId: connection.peerId,
             });
-            log('document not found locally for share policy check, accepting the remote document', {
+            log('document not found locally for share policy check', {
               peerId: connection.peerId,
               documentId: params.documentId,
-              remoteDocumentExists,
+              acceptDocument: remoteDocumentExists,
             });
             // If a document is not present locally return true if another peer claims to have it.
             // Simply returning true will add the peer to "generous peers list" for this document which will

package/src/db-host/echo-host.ts

@@ -33,6 +33,7 @@ import {
   type EchoReplicator,
   type EchoDataStats,
   type PeerIdProvider,
+  type RootDocumentSpaceKeyProvider,
 } from '../automerge';
 
 const INDEXER_CONFIG: IndexConfig = {
@@ -43,6 +44,7 @@ const INDEXER_CONFIG: IndexConfig = {
 export type EchoHostParams = {
   kv: LevelDB;
   peerIdProvider?: PeerIdProvider;
+  getSpaceKeyByRootDocumentId?: RootDocumentSpaceKeyProvider;
 };
 
 /**
@@ -60,7 +62,7 @@ export class EchoHost extends Resource {
   private readonly _spaceStateManager = new SpaceStateManager();
   private readonly _echoDataMonitor: EchoDataMonitor;
 
-  constructor({ kv, peerIdProvider }: EchoHostParams) {
+  constructor({ kv, peerIdProvider, getSpaceKeyByRootDocumentId }: EchoHostParams) {
     super();
 
     this._indexMetadataStore = new IndexMetadataStore({ db: kv.sublevel('index-metadata') });
@@ -72,6 +74,7 @@ export class EchoHost extends Resource {
       dataMonitor: this._echoDataMonitor,
       indexMetadataStore: this._indexMetadataStore,
       peerIdProvider,
+      getSpaceKeyByRootDocumentId,
     });
 
     this._indexer = new Indexer({

package/src/edge/echo-edge-replicator.ts

@@ -2,8 +2,7 @@
 // Copyright 2024 DXOS.org
 //
 
-import { Mutex, scheduleTask, scheduleMicroTask } from '@dxos/async';
-import * as A from '@dxos/automerge/automerge';
+import { Mutex, scheduleTask, scheduleMicroTask, Trigger } from '@dxos/async';
 import { cbor } from '@dxos/automerge/automerge-repo';
 import { Context, Resource } from '@dxos/context';
 import { randomUUID } from '@dxos/crypto';
@@ -184,9 +183,11 @@ type EdgeReplicatorConnectionsParams = {
   onRestartRequested: () => Promise<void>;
 };
 
+const MAX_INFLIGHT_REQUESTS = 5;
+
 class EdgeReplicatorConnection extends Resource implements ReplicatorConnection {
   private readonly _edgeConnection: EdgeConnection;
-  private _remotePeerId: string | null = null;
+  private readonly _remotePeerId: string | null = null;
   private readonly _targetServiceId: string;
   private readonly _spaceId: SpaceId;
   private readonly _context: EchoReplicatorContext;
@@ -195,6 +196,16 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
   private readonly _onRemoteDisconnected: () => Promise<void>;
   private readonly _onRestartRequested: () => void;
 
+  /**
+   * Prevents sending too many messages to edge over this connection so that we don't overwhelm
+   * a replicator durable object.
+   * inflightRequests counter is incremented on outgoing sync messages and decremented on incoming messages.
+   * The trigger is waiting while the counter is above MAX_INFLIGHT_REQUESTS.
+   * The counter can go negative because we receive edge-initiated sync messages on doc change broadcasts.
+   */
+  private _outgoingRequestsBarrier = new Trigger();
+  private _inflightRequests = 0;
+
   private _readableStreamController!: ReadableStreamDefaultController<AutomergeProtocolMessage>;
 
   public readable: ReadableStream<AutomergeProtocolMessage>;
@@ -213,7 +224,6 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
     this._edgeConnection = edgeConnection;
     this._spaceId = spaceId;
     this._context = context;
-
     // Generate a unique peer id for every connection.
     // This way automerge-repo will have separate sync states for every connection.
     // This is important because the previous connection might have had some messages that failed to deliver
@@ -225,6 +235,8 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
     this._onRemoteDisconnected = onRemoteDisconnected;
     this._onRestartRequested = onRestartRequested;
 
+    this._outgoingRequestsBarrier.wake();
+
     this.readable = new ReadableStream<AutomergeProtocolMessage>({
       start: (controller) => {
         this._readableStreamController = controller;
@@ -233,6 +245,13 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
 
     this.writable = new WritableStream<AutomergeProtocolMessage>({
       write: async (message: AutomergeProtocolMessage, controller) => {
+        await this._outgoingRequestsBarrier.wait();
+
+        this._inflightRequests++;
+        if (this._inflightRequests === MAX_INFLIGHT_REQUESTS) {
+          this._outgoingRequestsBarrier.reset();
+        }
+
         await this._sendMessage(message);
       },
     });
@@ -253,6 +272,9 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
   protected override async _close(): Promise<void> {
     log('close');
     this._readableStreamController.close();
+
+    this._outgoingRequestsBarrier.throw(new Error('Connection closed.'));
+
     await this._onRemoteDisconnected();
   }
 
@@ -272,9 +294,10 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
         peerId: this._remotePeerId as PeerId,
       });
 
-      log.info('document not found locally for share policy check, accepting the remote document', {
+      log.verbose('edge-replicator document not found locally for share policy check', {
         documentId: params.documentId,
-        remoteDocumentExists,
+        acceptDocument: remoteDocumentExists,
+        remoteId: this._remotePeerId,
       });
 
       // If a document is not present locally return true only if it already exists on edge.
@@ -290,7 +313,8 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
       return true;
     }
     const spaceId = getSpaceIdFromCollectionId(params.collectionId as CollectionId);
-    return spaceId === this._spaceId;
+    // Only sync collections of form space:id:rootDoc, edge ignores legacy space:id collections
+    return spaceId === this._spaceId && params.collectionId.split(':').length === 3;
   }
 
   private _onMessage(message: RouterMessage) {
@@ -299,15 +323,12 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
     }
 
     const payload = cbor.decode(message.payload!.value) as AutomergeProtocolMessage;
-    log('recv', () => {
-      const decodedData =
-        payload.type === 'sync' && payload.data
-          ? A.decodeSyncMessage(payload.data)
-          : payload.type === 'collection-state'
-            ? (payload as any).state
-            : payload;
-      return { from: message.serviceId, type: payload.type, decodedData };
+    log.verbose('edge replicator receive', {
+      type: payload.type,
+      documentId: payload.type === 'sync' && payload.documentId,
+      remoteId: this._remotePeerId,
     });
+
     // Fix the peer id.
     payload.senderId = this._remotePeerId! as PeerId;
     this._processMessage(payload);
@@ -322,6 +343,13 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
       return;
     }
 
+    if (message.type === 'sync') {
+      this._inflightRequests--;
+      if (this._inflightRequests === MAX_INFLIGHT_REQUESTS - 1) {
+        this._outgoingRequestsBarrier.wake();
+      }
+    }
+
     this._readableStreamController.enqueue(message);
   }
 
@@ -329,12 +357,12 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
     // Fix the peer id.
     (message as any).targetId = this._targetServiceId as PeerId;
 
-    log('send', {
+    log.verbose('edge replicator send', {
       type: message.type,
-      senderId: message.senderId,
-      targetId: (message as any).targetId,
-      documentId: (message as any).documentId,
+      documentId: message.type === 'sync' && message.documentId,
+      remoteId: this._remotePeerId,
     });
+
     const encoded = cbor.encode(message);
 
     await this._edgeConnection.send(

package/src/space/space-manager.ts

@@ -3,7 +3,8 @@
 //
 
 import { synchronized, trackLeaks, Trigger } from '@dxos/async';
-import { type DelegateInvitationCredential, type MemberInfo } from '@dxos/credentials';
+import { type AutomergeUrl, parseAutomergeUrl } from '@dxos/automerge/automerge-repo';
+import { getCredentialAssertion, type DelegateInvitationCredential, type MemberInfo } from '@dxos/credentials';
 import { failUndefined } from '@dxos/debug';
 import { type FeedStore } from '@dxos/feed-store';
 import { PublicKey } from '@dxos/keys';
@@ -168,4 +169,19 @@ export class SpaceManager {
       await protocol.stop();
     }
   }
+
+  public findSpaceByRootDocumentId(documentId: string): Space | undefined {
+    return [...this._spaces.values()].find((space) => {
+      return space.spaceState.credentials.some((credential) => {
+        const assertion = getCredentialAssertion(credential);
+        if (assertion['@type'] !== 'dxos.halo.credentials.Epoch') {
+          return false;
+        }
+        if (!assertion?.automergeRoot) {
+          return false;
+        }
+        return parseAutomergeUrl(assertion.automergeRoot as AutomergeUrl).documentId === documentId;
+      });
+    });
+  }
 }