@dxos/echo-pipeline 0.6.3-main.9e4e207 → 0.6.3-next.2f65b78

package/src/automerge/echo-network-adapter.test.ts

@@ -103,11 +103,7 @@ describe('EchoNetworkAdapter', () => {
   });
 
   const createConnectedAdapter = async (replicator: MeshEchoReplicator) => {
-    const adapter = new EchoNetworkAdapter({
-      getContainingSpaceForDocument: async () => null,
-      onCollectionStateQueried: () => {},
-      onCollectionStateReceived: () => {},
-    });
+    const adapter = new EchoNetworkAdapter({ getContainingSpaceForDocument: async () => null });
     adapter.connect(PEER_ID);
     await adapter.open();
     afterTest(() => adapter.close());

package/src/automerge/echo-network-adapter.ts

@@ -3,30 +3,16 @@
 //
 
 import { synchronized, Trigger } from '@dxos/async';
-import { NetworkAdapter, type Message, type PeerId, type PeerMetadata } from '@dxos/automerge/automerge-repo';
+import { type Message, NetworkAdapter, type PeerId, type PeerMetadata } from '@dxos/automerge/automerge-repo';
 import { LifecycleState } from '@dxos/context';
 import { invariant } from '@dxos/invariant';
 import { type PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
-import { nonNullable } from '@dxos/util';
-
-import {
-  type EchoReplicator,
-  type ReplicatorConnection,
-  type ShouldAdvertiseParams,
-  type ShouldSyncCollectionParams,
-} from './echo-replicator';
-import {
-  isCollectionQueryMessage,
-  isCollectionStateMessage,
-  type CollectionQueryMessage,
-  type CollectionStateMessage,
-} from './network-protocol';
+
+import { type EchoReplicator, type ReplicatorConnection, type ShouldAdvertiseParams } from './echo-replicator';
 
 export type EchoNetworkAdapterParams = {
   getContainingSpaceForDocument: (documentId: string) => Promise<PublicKey | null>;
-  onCollectionStateQueried: (collectionId: string, peerId: PeerId) => void;
-  onCollectionStateReceived: (collectionId: string, peerId: PeerId, state: unknown) => void;
 };
 
 /**
@@ -133,47 +119,6 @@ export class EchoNetworkAdapter extends NetworkAdapter {
     return connection.connection.shouldAdvertise(params);
   }
 
-  shouldSyncCollection(peerId: PeerId, params: ShouldSyncCollectionParams): boolean {
-    const connection = this._connections.get(peerId);
-    if (!connection) {
-      return false;
-    }
-
-    return connection.connection.shouldSyncCollection(params);
-  }
-
-  queryCollectionState(collectionId: string, targetId: PeerId): void {
-    const message: CollectionQueryMessage = {
-      type: 'collection-query',
-      senderId: this.peerId as PeerId,
-      targetId,
-      collectionId,
-    };
-    this.send(message);
-  }
-
-  sendCollectionState(collectionId: string, targetId: PeerId, state: unknown): void {
-    const message: CollectionStateMessage = {
-      type: 'collection-state',
-      senderId: this.peerId as PeerId,
-      targetId,
-      collectionId,
-      state,
-    };
-    this.send(message);
-  }
-
-  // TODO(dmaretskyi): Remove.
-  getPeersInterestedInCollection(collectionId: string): PeerId[] {
-    return Array.from(this._connections.values())
-      .map((connection) => {
-        return connection.connection.shouldSyncCollection({ collectionId })
-          ? (connection.connection.peerId as PeerId)
-          : null;
-      })
-      .filter(nonNullable);
-  }
-
   private _onConnectionOpen(connection: ReplicatorConnection) {
     log('Connection opened', { peerId: connection.peerId });
     invariant(!this._connections.has(connection.peerId as PeerId));
@@ -191,7 +136,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
             break;
           }
 
-          this._onMessage(value);
+          this.emit('message', value);
         }
       } catch (err) {
         if (connectionEntry.isOpen) {
@@ -204,16 +149,6 @@ export class EchoNetworkAdapter extends NetworkAdapter {
     this._emitPeerCandidate(connection);
   }
 
-  private _onMessage(message: Message) {
-    if (isCollectionQueryMessage(message)) {
-      this._params.onCollectionStateQueried(message.collectionId, message.senderId);
-    } else if (isCollectionStateMessage(message)) {
-      this._params.onCollectionStateReceived(message.collectionId, message.senderId, message.state);
-    } else {
-      this.emit('message', message);
-    }
-  }
-
   /**
    * Trigger doc-synchronizer shared documents set recalculation. Happens on peer-candidate.
    * TODO(y): replace with a proper API call when sharePolicy update becomes supported by automerge-repo

package/src/automerge/echo-replicator.ts

@@ -51,17 +51,8 @@ export interface ReplicatorConnection {
    * The remote peer can still request the document by its id bypassing this check.
    */
   shouldAdvertise(params: ShouldAdvertiseParams): Promise<boolean>;
-
-  /**
-   * @returns true if the collection should be synced to this peer.
-   */
-  shouldSyncCollection(params: ShouldSyncCollectionParams): boolean;
 }
 
 export type ShouldAdvertiseParams = {
   documentId: string;
 };
-
-export type ShouldSyncCollectionParams = {
-  collectionId: string;
-};

package/src/automerge/heads-store.ts

@@ -26,11 +26,14 @@ export class HeadsStore {
     });
   }
 
-  // TODO(dmaretskyi): Make batched.
-  async getHeads(documentIds: DocumentId[]): Promise<Array<Heads | undefined>> {
-    return this._db.getMany<DocumentId, Heads>(documentIds, {
-      keyEncoding: 'utf8',
-      valueEncoding: headsEncoding,
-    });
+  async getHeads(documentId: DocumentId): Promise<Heads | undefined> {
+    try {
+      return await this._db.get<DocumentId, Heads>(documentId, { keyEncoding: 'utf8', valueEncoding: headsEncoding });
+    } catch (err: any) {
+      if (err.notFound) {
+        return undefined;
+      }
+      throw err;
+    }
   }
 }

package/src/automerge/index.ts

@@ -6,5 +6,3 @@ export * from './automerge-host';
 export * from './leveldb-storage-adapter';
 export * from './mesh-echo-replicator';
 export * from './echo-replicator';
-export { diffCollectionState } from './collection-synchronizer';
-export * from './space-collection';

package/src/automerge/mesh-echo-replicator-connection.ts

@@ -9,7 +9,7 @@ import { type PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 import { AutomergeReplicator, type AutomergeReplicatorFactory } from '@dxos/teleport-extension-automerge-replicator';
 
-import type { ReplicatorConnection, ShouldAdvertiseParams, ShouldSyncCollectionParams } from './echo-replicator';
+import type { ReplicatorConnection, ShouldAdvertiseParams } from './echo-replicator';
 
 const DEFAULT_FACTORY: AutomergeReplicatorFactory = (params) => new AutomergeReplicator(...params);
 
@@ -18,7 +18,6 @@ export type MeshReplicatorConnectionParams = {
   onRemoteConnected: () => void;
   onRemoteDisconnected: () => void;
   shouldAdvertise: (params: ShouldAdvertiseParams) => Promise<boolean>;
-  shouldSyncCollection: (params: ShouldSyncCollectionParams) => boolean;
   replicatorFactory?: AutomergeReplicatorFactory;
 };
 
@@ -113,10 +112,6 @@ export class MeshReplicatorConnection extends Resource implements ReplicatorConn
     return this._params.shouldAdvertise(params);
   }
 
-  shouldSyncCollection(params: ShouldSyncCollectionParams): boolean {
-    return this._params.shouldSyncCollection(params);
-  }
-
   /**
    * Start exchanging messages with the remote peer.
    * Call after the remote peer has connected.

package/src/automerge/mesh-echo-replicator.ts

@@ -3,18 +3,16 @@
 //
 
 import { invariant } from '@dxos/invariant';
-import { PublicKey, type SpaceId } from '@dxos/keys';
+import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 import {
   type AutomergeReplicator,
   type AutomergeReplicatorFactory,
 } from '@dxos/teleport-extension-automerge-replicator';
-import { ComplexSet, defaultMap } from '@dxos/util';
+import { ComplexMap, ComplexSet, defaultMap } from '@dxos/util';
 
 import { type EchoReplicator, type EchoReplicatorContext, type ShouldAdvertiseParams } from './echo-replicator';
 import { MeshReplicatorConnection } from './mesh-echo-replicator-connection';
-import { getSpaceIdFromCollectionId } from './space-collection';
-import { createIdFromSpaceKey } from '../space';
 
 // TODO(dmaretskyi): Move out of @dxos/echo-pipeline.
 
@@ -29,9 +27,9 @@ export class MeshEchoReplicator implements EchoReplicator {
   private readonly _connectionsPerPeer = new Map<string, MeshReplicatorConnection>();
 
   /**
-   * spaceId -> deviceKey[]
+   * spaceKey -> deviceKey[]
    */
-  private readonly _authorizedDevices = new Map<SpaceId, ComplexSet<PublicKey>>();
+  private readonly _authorizedDevices = new ComplexMap<PublicKey, ComplexSet<PublicKey>>(PublicKey.hash);
 
   private _context: EchoReplicatorContext | null = null;
 
@@ -90,9 +88,7 @@ export class MeshEchoReplicator implements EchoReplicator {
             return false;
           }
 
-          const spaceId = await createIdFromSpaceKey(spaceKey);
-
-          const authorizedDevices = this._authorizedDevices.get(spaceId);
+          const authorizedDevices = this._authorizedDevices.get(spaceKey);
 
           if (!connection.remoteDeviceKey) {
             log('device key not found for share policy check', {
@@ -117,32 +113,15 @@ export class MeshEchoReplicator implements EchoReplicator {
           return false;
         }
       },
-      shouldSyncCollection: ({ collectionId }) => {
-        const spaceId = getSpaceIdFromCollectionId(collectionId);
-
-        const authorizedDevices = this._authorizedDevices.get(spaceId);
-
-        if (!connection.remoteDeviceKey) {
-          log('device key not found for collection sync check', {
-            peerId: connection.peerId,
-            collectionId,
-          });
-          return false;
-        }
-
-        const isAuthorized = authorizedDevices?.has(connection.remoteDeviceKey) ?? false;
-        return isAuthorized;
-      },
     });
     this._connections.add(connection);
 
     return connection.replicatorExtension;
   }
 
-  async authorizeDevice(spaceKey: PublicKey, deviceKey: PublicKey) {
+  authorizeDevice(spaceKey: PublicKey, deviceKey: PublicKey) {
     log('authorizeDevice', { spaceKey, deviceKey });
-    const spaceId = await createIdFromSpaceKey(spaceKey);
-    defaultMap(this._authorizedDevices, spaceId, () => new ComplexSet(PublicKey.hash)).add(deviceKey);
+    defaultMap(this._authorizedDevices, spaceKey, () => new ComplexSet(PublicKey.hash)).add(deviceKey);
     for (const connection of this._connections) {
       if (connection.remoteDeviceKey && connection.remoteDeviceKey.equals(deviceKey)) {
         if (this._connectionsPerPeer.has(connection.peerId)) {

package/src/db-host/data-service.ts

@@ -5,10 +5,10 @@
 import { type DocumentId } from '@dxos/automerge/automerge-repo';
 import { type RequestOptions, Stream } from '@dxos/codec-protobuf';
 import { invariant } from '@dxos/invariant';
-import { SpaceId } from '@dxos/keys';
 import { log } from '@dxos/log';
 import {
   type DataService,
+  type DocHeadsList,
   type FlushRequest,
   type SubscribeRequest,
   type BatchedDocumentUpdates,
@@ -18,12 +18,10 @@ import {
   type ReIndexHeadsRequest,
   type WaitUntilHeadsReplicatedRequest,
   type UpdateRequest,
-  type GetSpaceSyncStateRequest,
-  type SpaceSyncState,
 } from '@dxos/protocols/proto/dxos/echo/service';
 
 import { DocumentsSynchronizer } from './documents-synchronizer';
-import { deriveCollectionIdFromSpaceId, type AutomergeHost } from '../automerge';
+import { type AutomergeHost } from '../automerge';
 
 export type DataServiceParams = {
   automergeHost: AutomergeHost;
@@ -93,14 +91,18 @@ export class DataServiceImpl implements DataService {
   }
 
   async getDocumentHeads(request: GetDocumentHeadsRequest): Promise<GetDocumentHeadsResponse> {
-    const documentIds = request.documentIds;
-    if (!documentIds) {
-      return { heads: { entries: [] } };
-    }
-    const heads = await this._automergeHost.getHeads(documentIds as DocumentId[]);
+    const entries = await Promise.all(
+      request.documentIds?.map(async (documentId): Promise<DocHeadsList.Entry> => {
+        const heads = await this._automergeHost.getHeads(documentId as DocumentId);
+        return {
+          documentId,
+          heads,
+        };
+      }) ?? [],
+    );
     return {
       heads: {
-        entries: heads.map((heads, idx) => ({ documentId: documentIds[idx], heads })),
+        entries,
       },
     };
   }
@@ -119,20 +121,4 @@ export class DataServiceImpl implements DataService {
   async updateIndexes() {
     await this._updateIndexes();
   }
-
-  async getSpaceSyncState(
-    request: GetSpaceSyncStateRequest,
-    options?: RequestOptions | undefined,
-  ): Promise<SpaceSyncState> {
-    invariant(SpaceId.isValid(request.spaceId));
-    const collectionId = deriveCollectionIdFromSpaceId(request.spaceId);
-    const state = await this._automergeHost.getCollectionSyncState(collectionId);
-
-    return {
-      peers: state.peers.map((peer) => ({
-        peerId: peer.peerId,
-        documentsToReconcile: peer.differentDocuments,
-      })),
-    };
-  }
 }

package/src/testing/index.ts

@@ -6,4 +6,3 @@ export * from './change-metadata';
 export * from './test-agent-builder';
 export * from './test-feed-builder';
 export * from './test-network-adapter';
-export * from './test-replicator';