@dxos/echo-pipeline 0.6.5 → 0.6.6-staging.23d123d

package/src/automerge/echo-network-adapter.test.ts

@@ -105,6 +105,7 @@ describe('EchoNetworkAdapter', () => {
   const createConnectedAdapter = async (replicator: MeshEchoReplicator) => {
     const adapter = new EchoNetworkAdapter({
       getContainingSpaceForDocument: async () => null,
+      isDocumentInRemoteCollection: async () => true,
       onCollectionStateQueried: () => {},
       onCollectionStateReceived: () => {},
     });

package/src/automerge/echo-network-adapter.ts

@@ -12,6 +12,7 @@ import { nonNullable } from '@dxos/util';
 
 import {
   type EchoReplicator,
+  type RemoteDocumentExistenceCheckParams,
   type ReplicatorConnection,
   type ShouldAdvertiseParams,
   type ShouldSyncCollectionParams,
@@ -22,6 +23,7 @@ import {
   type CollectionQueryMessage,
   type CollectionStateMessage,
 } from './network-protocol';
+import { createIdFromSpaceKey } from '../space';
 
 export interface NetworkDataMonitor {
   recordPeerConnected(peerId: string): void;
@@ -33,6 +35,7 @@ export interface NetworkDataMonitor {
 
 export type EchoNetworkAdapterParams = {
   getContainingSpaceForDocument: (documentId: string) => Promise<PublicKey | null>;
+  isDocumentInRemoteCollection: (params: RemoteDocumentExistenceCheckParams) => Promise<boolean>;
   onCollectionStateQueried: (collectionId: string, peerId: PeerId) => void;
   onCollectionStateReceived: (collectionId: string, peerId: PeerId, state: unknown) => void;
   monitor?: NetworkDataMonitor;
@@ -111,7 +114,12 @@ export class EchoNetworkAdapter extends NetworkAdapter {
       onConnectionOpen: this._onConnectionOpen.bind(this),
       onConnectionClosed: this._onConnectionClosed.bind(this),
       onConnectionAuthScopeChanged: this._onConnectionAuthScopeChanged.bind(this),
+      isDocumentInRemoteCollection: this._params.isDocumentInRemoteCollection,
       getContainingSpaceForDocument: this._params.getContainingSpaceForDocument,
+      getContainingSpaceIdForDocument: async (documentId) => {
+        const key = await this._params.getContainingSpaceForDocument(documentId);
+        return key ? createIdFromSpaceKey(key) : null;
+      },
     });
   }
 

package/src/automerge/echo-replicator.ts

@@ -3,7 +3,7 @@
 //
 
 import { type Message } from '@dxos/automerge/automerge-repo';
-import { type PublicKey } from '@dxos/keys';
+import { type PublicKey, type SpaceId } from '@dxos/keys';
 
 export interface EchoReplicator {
   /**
@@ -23,7 +23,12 @@ export interface EchoReplicatorContext {
    */
   get peerId(): string;
 
+  /**
+   * @deprecated Use `getContainingSpaceIdForDocument`.
+   */
   getContainingSpaceForDocument(documentId: string): Promise<PublicKey | null>;
+  getContainingSpaceIdForDocument(documentId: string): Promise<SpaceId | null>;
+  isDocumentInRemoteCollection(params: RemoteDocumentExistenceCheckParams): Promise<boolean>;
 
   onConnectionOpen(connection: ReplicatorConnection): void;
   onConnectionClosed(connection: ReplicatorConnection): void;
@@ -65,3 +70,8 @@ export type ShouldAdvertiseParams = {
 export type ShouldSyncCollectionParams = {
   collectionId: string;
 };
+
+export type RemoteDocumentExistenceCheckParams = {
+  peerId: string;
+  documentId: string;
+};

package/src/automerge/mesh-echo-replicator-connection.ts

@@ -2,6 +2,7 @@
 // Copyright 2024 DXOS.org
 //
 
+import * as A from '@dxos/automerge/automerge';
 import { cbor, type Message } from '@dxos/automerge/automerge-repo';
 import { Resource } from '@dxos/context';
 import { invariant } from '@dxos/invariant';
@@ -47,6 +48,7 @@ export class MeshReplicatorConnection extends Resource implements ReplicatorConn
       write: async (message: Message, controller) => {
         invariant(this._isEnabled, 'Writing to a disabled connection');
         try {
+          logSendSync(message);
           await this.replicatorExtension.sendSyncMessage({ payload: cbor.encode(message) });
         } catch (err) {
           controller.error(err);
@@ -133,3 +135,19 @@ export class MeshReplicatorConnection extends Resource implements ReplicatorConn
     this._isEnabled = false;
   }
 }
+
+const logSendSync = (message: Message) => {
+  log('sendSyncMessage', () => {
+    const decodedSyncMessage = message.type === 'sync' && message.data ? A.decodeSyncMessage(message.data) : undefined;
+    return {
+      sync: decodedSyncMessage && {
+        headsLength: decodedSyncMessage.heads.length,
+        requesting: decodedSyncMessage.need.length > 0,
+        sendingChanges: decodedSyncMessage.changes.length > 0,
+      },
+      type: message.type,
+      from: message.senderId,
+      to: message.targetId,
+    };
+  });
+};

package/src/automerge/mesh-echo-replicator.ts

@@ -83,11 +83,19 @@ export class MeshEchoReplicator implements EchoReplicator {
         try {
           const spaceKey = await this._context.getContainingSpaceForDocument(params.documentId);
           if (!spaceKey) {
-            log('space key not found for share policy check', {
+            const remoteDocumentExists = await this._context.isDocumentInRemoteCollection({
+              documentId: params.documentId,
+              peerId: connection.peerId,
+            });
+            log('document not found locally for share policy check, accepting the remote document', {
               peerId: connection.peerId,
               documentId: params.documentId,
+              remoteDocumentExists,
             });
-            return false;
+            // If a document is not present locally return true if another peer claims to have it.
+            // Simply returning true will add the peer to "generous peers list" for this document which will
+            // start replication of the document after we receive, even if the peer is not in the corresponding space.
+            return remoteDocumentExists;
           }
 
           const spaceId = await createIdFromSpaceKey(spaceKey);

package/src/automerge/network-protocol.ts

@@ -2,44 +2,18 @@
 // Copyright 2024 DXOS.org
 //
 
-import type { Message, PeerId } from '@dxos/automerge/automerge-repo';
+import type { Message } from '@dxos/automerge/automerge-repo';
+import {
+  type CollectionQueryMessage,
+  type CollectionStateMessage,
+  MESSAGE_TYPE_COLLECTION_QUERY,
+  MESSAGE_TYPE_COLLECTION_STATE,
+} from '@dxos/protocols';
 
-export const MESSAGE_TYPE_COLLECTION_QUERY = 'collection-query';
-
-export type CollectionQueryMessage = {
-  type: typeof MESSAGE_TYPE_COLLECTION_QUERY;
-  senderId: PeerId;
-  targetId: PeerId;
-  collectionId: string;
-
-  /**
-   * Identifier of the current state.
-   * Remote peer will skip sending the state if it has the same tag.
-   */
-  stateTag?: string;
-};
+export { type CollectionStateMessage, type CollectionQueryMessage };
 
 export const isCollectionQueryMessage = (message: Message): message is CollectionQueryMessage =>
   message.type === MESSAGE_TYPE_COLLECTION_QUERY;
 
-export const MESSAGE_TYPE_COLLECTION_STATE = 'collection-state';
-
-export type CollectionStateMessage = {
-  type: typeof MESSAGE_TYPE_COLLECTION_STATE;
-  senderId: PeerId;
-  targetId: PeerId;
-  collectionId: string;
-
-  /**
-   *  State representation is implementation-defined.
-   */
-  state?: unknown;
-
-  /**
-   * Identifier of the current state.
-   */
-  stateTag?: string;
-};
-
 export const isCollectionStateMessage = (message: Message): message is CollectionStateMessage =>
   message.type === MESSAGE_TYPE_COLLECTION_STATE;

package/src/space/space-manager.ts

@@ -34,6 +34,8 @@ export type SpaceManagerParams = {
   snapshotStore: SnapshotStore;
 
   blobStore: BlobStore;
+
+  disableP2pReplication?: boolean;
 };
 
 export type ConstructSpaceParams = {
@@ -68,14 +70,23 @@ export class SpaceManager {
   private readonly _snapshotStore: SnapshotStore;
   private readonly _blobStore: BlobStore;
   private readonly _instanceId = PublicKey.random().toHex();
-
-  constructor({ feedStore, networkManager, metadataStore, snapshotStore, blobStore }: SpaceManagerParams) {
+  private readonly _disableP2pReplication: boolean;
+
+  constructor({
+    feedStore,
+    networkManager,
+    metadataStore,
+    snapshotStore,
+    blobStore,
+    disableP2pReplication,
+  }: SpaceManagerParams) {
     // TODO(burdon): Assert.
     this._feedStore = feedStore;
     this._networkManager = networkManager;
     this._metadataStore = metadataStore;
     this._snapshotStore = snapshotStore;
     this._blobStore = blobStore;
+    this._disableP2pReplication = disableP2pReplication ?? false;
   }
 
   // TODO(burdon): Remove.
@@ -115,6 +126,7 @@ export class SpaceManager {
       onSessionAuth: onAuthorizedConnection,
       onAuthFailure,
       blobStore: this._blobStore,
+      disableP2pReplication: this._disableP2pReplication,
     });
     const snapshotManager = new SnapshotManager(this._snapshotStore, this._blobStore, protocol.blobSync);
 
@@ -157,6 +169,7 @@ export class SpaceManager {
       },
       onAuthFailure: (session: Teleport) => session.close(),
       blobStore: this._blobStore,
+      disableP2pReplication: this._disableP2pReplication,
     });
 
     try {

package/src/space/space-protocol.ts

@@ -20,7 +20,7 @@ import { type MuxerStats, Teleport } from '@dxos/teleport';
 import { type BlobStore, BlobSync } from '@dxos/teleport-extension-object-sync';
 import { ReplicatorExtension } from '@dxos/teleport-extension-replicator';
 import { trace } from '@dxos/tracing';
-import { ComplexMap } from '@dxos/util';
+import { CallbackCollection, ComplexMap, type AsyncCallback } from '@dxos/util';
 
 import { AuthExtension, type AuthProvider, type AuthVerifier } from './auth';
 
@@ -41,12 +41,16 @@ export type SpaceProtocolOptions = {
 
   blobStore: BlobStore;
 
+  onFeed?: (feed: FeedWrapper<FeedMessage>) => Promise<void>;
+
   /**
    * Called when new session is authenticated.
    * Additional extensions can be added here.
    */
   onSessionAuth?: (session: Teleport) => void;
   onAuthFailure?: (session: Teleport) => void;
+
+  disableP2pReplication?: boolean;
 };
 
 /**
@@ -61,6 +65,8 @@ export class SpaceProtocol {
 
   public readonly blobSync: BlobSync;
 
+  private readonly _disableP2pReplication: boolean;
+
   @logInfo
   @trace.info()
   private readonly _topic: Promise<PublicKey>;
@@ -79,6 +85,8 @@ export class SpaceProtocol {
 
   private _connection?: SwarmConnection;
 
+  public readonly feedAdded = new CallbackCollection<AsyncCallback<FeedWrapper<FeedMessage>>>();
+
   get sessions(): ReadonlyMap<PublicKey, SpaceProtocolSession> {
     return this._sessions;
   }
@@ -92,7 +100,15 @@ export class SpaceProtocol {
     return this._swarmIdentity.peerKey;
   }
 
-  constructor({ topic, swarmIdentity, networkManager, onSessionAuth, onAuthFailure, blobStore }: SpaceProtocolOptions) {
+  constructor({
+    topic,
+    swarmIdentity,
+    networkManager,
+    onSessionAuth,
+    onAuthFailure,
+    blobStore,
+    disableP2pReplication,
+  }: SpaceProtocolOptions) {
     this._spaceKey = topic;
     this._networkManager = networkManager;
     this._swarmIdentity = swarmIdentity;
@@ -102,16 +118,20 @@ export class SpaceProtocol {
 
     // TODO(burdon): Async race condition? Move to start?
     this._topic = subtleCrypto.digest('SHA-256', topic.asBuffer()).then(discoveryKey).then(PublicKey.from);
+
+    this._disableP2pReplication = disableP2pReplication ?? false;
   }
 
   // TODO(burdon): Create abstraction for Space (e.g., add keys and have provider).
-  addFeed(feed: FeedWrapper<FeedMessage>) {
+  async addFeed(feed: FeedWrapper<FeedMessage>) {
     log('addFeed', { key: feed.key });
 
     this._feeds.add(feed);
     for (const session of this._sessions.values()) {
       session.replicator.addFeed(feed);
     }
+
+    await this.feedAdded.callSerial(feed);
   }
 
   // TODO(burdon): Rename open? Common open/close interfaces for all services?
@@ -160,6 +180,7 @@ export class SpaceProtocol {
         onSessionAuth: this._onSessionAuth,
         onAuthFailure: this._onAuthFailure,
         blobSync: this.blobSync,
+        disableP2pReplication: this._disableP2pReplication,
       });
       this._sessions.set(wireParams.remotePeerId, session);
 
@@ -185,6 +206,8 @@ export type SpaceProtocolSessionParams = {
   onSessionAuth?: (session: Teleport) => void;
 
   onAuthFailure?: (session: Teleport) => void;
+
+  disableP2pReplication?: boolean;
 };
 
 export enum AuthStatus {
@@ -201,6 +224,8 @@ export class SpaceProtocolSession implements WireProtocol {
   @logInfo
   private readonly _wireParams: WireProtocolParams;
 
+  private readonly _disableP2pReplication: boolean;
+
   private readonly _onSessionAuth?: (session: Teleport) => void;
   private readonly _onAuthFailure?: (session: Teleport) => void;
   private readonly _swarmIdentity: SwarmIdentity;
@@ -223,7 +248,14 @@ export class SpaceProtocolSession implements WireProtocol {
   }
 
   // TODO(dmaretskyi): Allow to pass in extra extensions.
-  constructor({ wireParams, swarmIdentity, onSessionAuth, onAuthFailure, blobSync }: SpaceProtocolSessionParams) {
+  constructor({
+    wireParams,
+    swarmIdentity,
+    onSessionAuth,
+    onAuthFailure,
+    blobSync,
+    disableP2pReplication,
+  }: SpaceProtocolSessionParams) {
     this._wireParams = wireParams;
     this._swarmIdentity = swarmIdentity;
     this._onSessionAuth = onSessionAuth;
@@ -231,6 +263,8 @@ export class SpaceProtocolSession implements WireProtocol {
     this._blobSync = blobSync;
 
     this._teleport = new Teleport(wireParams);
+
+    this._disableP2pReplication = disableP2pReplication ?? false;
   }
 
   get stream() {
@@ -256,7 +290,10 @@ export class SpaceProtocolSession implements WireProtocol {
         },
       }),
     );
-    this._teleport.addExtension('dxos.mesh.teleport.replicator', this.replicator);
+
+    if (!this._disableP2pReplication) {
+      this._teleport.addExtension('dxos.mesh.teleport.replicator', this.replicator);
+    }
     this._teleport.addExtension('dxos.mesh.teleport.blobsync', this._blobSync.createExtension());
   }
 

package/src/space/space.ts

@@ -2,7 +2,7 @@
 // Copyright 2022 DXOS.org
 //
 
-import { Event, synchronized, trackLeaks } from '@dxos/async';
+import { Event, scheduleMicroTask, synchronized, trackLeaks } from '@dxos/async';
 import { type Context, LifecycleState, Resource } from '@dxos/context';
 import { type DelegateInvitationCredential, type FeedInfo, type MemberInfo } from '@dxos/credentials';
 import { subtleCrypto } from '@dxos/crypto';
@@ -93,8 +93,8 @@ export class Space extends Resource {
       const sparse = info.assertion.designation === AdmittedFeed.Designation.DATA;
 
       if (!info.key.equals(params.genesisFeed.key)) {
-        queueMicrotask(async () => {
-          this.protocol.addFeed(await params.feedProvider(info.key, { sparse }));
+        scheduleMicroTask(this._ctx, async () => {
+          await this.protocol.addFeed(await params.feedProvider(info.key, { sparse }));
         });
       }
     });
@@ -119,7 +119,6 @@ export class Space extends Resource {
 
     // Start replicating the genesis feed.
     this.protocol = params.protocol;
-    this.protocol.addFeed(params.genesisFeed);
   }
 
   @logInfo
@@ -192,6 +191,7 @@ export class Space extends Resource {
     // Order is important.
     await this._controlPipeline.start();
     await this.protocol.start();
+    await this.protocol.addFeed(await this._feedProvider(this._genesisFeedKey));
 
     log('opened');
   }

package/src/testing/test-network-adapter.ts

@@ -7,13 +7,15 @@ import { type Message, NetworkAdapter, type PeerId } from '@dxos/automerge/autom
 import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 
+export type TestConnectionStateProvider = () => 'on' | 'off';
+
 export class TestAdapter extends NetworkAdapter {
-  static createPair() {
+  static createPair(connectionStateProvider: TestConnectionStateProvider = () => 'on') {
     const adapter1: TestAdapter = new TestAdapter({
-      send: (message: Message) => sleep(10).then(() => adapter2.receive(message)),
+      send: (message: Message) => connectionStateProvider() === 'on' && sleep(10).then(() => adapter2.receive(message)),
     });
     const adapter2: TestAdapter = new TestAdapter({
-      send: (message: Message) => sleep(10).then(() => adapter1.receive(message)),
+      send: (message: Message) => connectionStateProvider() === 'on' && sleep(10).then(() => adapter1.receive(message)),
     });
 
     return [adapter1, adapter2];