@dxos/echo-pipeline 0.5.2 → 0.5.3-main.056e7da

package/src/automerge/echo-network-adapter.ts

@@ -6,10 +6,15 @@ import { Trigger, synchronized } from '@dxos/async';
 import { type Message, NetworkAdapter, type PeerId, type PeerMetadata } from '@dxos/automerge/automerge-repo';
 import { LifecycleState } from '@dxos/context';
 import { invariant } from '@dxos/invariant';
+import { type PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 
 import { type EchoReplicator, type ReplicatorConnection, type ShouldAdvertizeParams } from './echo-replicator';
 
+export type EchoNetworkAdapterParams = {
+  getContainingSpaceForDocument: (documentId: string) => Promise<PublicKey | null>;
+};
+
 /**
  * Manages a set of {@link EchoReplicator} instances.
  */
@@ -22,6 +27,10 @@ export class EchoNetworkAdapter extends NetworkAdapter {
   private _lifecycleState: LifecycleState = LifecycleState.CLOSED;
   private readonly _connected = new Trigger();
 
+  constructor(private readonly _params: EchoNetworkAdapterParams) {
+    super();
+  }
+
   override connect(peerId: PeerId, peerMetadata?: PeerMetadata | undefined): void {
     this.peerId = peerId;
     this.peerMetadata = peerMetadata;
@@ -51,6 +60,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
     invariant(this._lifecycleState === LifecycleState.CLOSED);
     this._lifecycleState = LifecycleState.OPEN;
 
+    log('emit ready');
     this.emit('ready', {
       network: this,
     });
@@ -74,20 +84,26 @@ export class EchoNetworkAdapter extends NetworkAdapter {
 
   @synchronized
   async addReplicator(replicator: EchoReplicator) {
+    invariant(this._lifecycleState === LifecycleState.OPEN);
     invariant(this.peerId);
     invariant(!this._replicators.has(replicator));
 
+    this._replicators.add(replicator);
     await replicator.connect({
       peerId: this.peerId,
       onConnectionOpen: this._onConnectionOpen.bind(this),
       onConnectionClosed: this._onConnectionClosed.bind(this),
+      getContainingSpaceForDocument: this._params.getContainingSpaceForDocument,
     });
   }
 
   @synchronized
   async removeReplicator(replicator: EchoReplicator) {
+    invariant(this._lifecycleState === LifecycleState.OPEN);
     invariant(this._replicators.has(replicator));
+    ('');
     await replicator.disconnect();
+    this._replicators.delete(replicator);
   }
 
   async shouldAdvertize(peerId: PeerId, params: ShouldAdvertizeParams): Promise<boolean> {
@@ -100,6 +116,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
   }
 
   private _onConnectionOpen(connection: ReplicatorConnection) {
+    log('Connection opened', { peerId: connection.peerId });
     invariant(!this._connections.has(connection.peerId as PeerId));
     const reader = connection.readable.getReader();
     const writer = connection.writable.getWriter();
@@ -124,6 +141,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
       }
     });
 
+    log('emit peer-candidate', { peerId: connection.peerId });
     this.emit('peer-candidate', {
       peerId: connection.peerId as PeerId,
       peerMetadata: {
@@ -134,6 +152,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
   }
 
   private _onConnectionClosed(connection: ReplicatorConnection) {
+    log('Connection closed', { peerId: connection.peerId });
     const entry = this._connections.get(connection.peerId as PeerId);
     invariant(entry);
 

package/src/automerge/echo-replicator.ts

@@ -3,6 +3,7 @@
 //
 
 import { type Message } from '@dxos/automerge/automerge-repo';
+import { type PublicKey } from '@dxos/keys';
 
 export interface EchoReplicator {
   /**
@@ -25,6 +26,8 @@ export interface EchoReplicatorContext {
   onConnectionOpen(connection: ReplicatorConnection): void;
 
   onConnectionClosed(connection: ReplicatorConnection): void;
+
+  getContainingSpaceForDocument(documentId: string): Promise<PublicKey | null>;
 }
 
 export interface ReplicatorConnection {

package/src/automerge/index.ts

@@ -7,5 +7,5 @@ export * from './automerge-storage-adapter';
 export * from './automerge-doc-loader';
 export * from './leveldb-storage-adapter';
 export * from './local-host-network-adapter';
-export * from './mesh-network-adapter';
+export * from './mesh-echo-replicator';
 export * from './echo-replicator';

package/src/automerge/leveldb-storage-adapter.ts

@@ -6,11 +6,11 @@ import { type MixedEncoding } from 'level-transcoder';
 
 import { type StorageAdapterInterface, type Chunk, type StorageKey } from '@dxos/automerge/automerge-repo';
 import { LifecycleState, Resource } from '@dxos/context';
-import { type BatchLevel, type SubLevelDB } from '@dxos/kv-store';
+import { type BatchLevel, type SublevelDB } from '@dxos/kv-store';
 import { type MaybePromise } from '@dxos/util';
 
 export type LevelDBStorageAdapterParams = {
-  db: SubLevelDB;
+  db: SublevelDB;
   callbacks?: StorageCallbacks;
 };
 

package/src/automerge/mesh-echo-replicator.ts

@@ -0,0 +1,231 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { type Message, cbor } from '@dxos/automerge/automerge-repo';
+import { Resource } from '@dxos/context';
+import { invariant } from '@dxos/invariant';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { AutomergeReplicator } from '@dxos/teleport-extension-automerge-replicator';
+import { ComplexMap, ComplexSet, defaultMap } from '@dxos/util';
+
+import {
+  type EchoReplicator,
+  type EchoReplicatorContext,
+  type ReplicatorConnection,
+  type ShouldAdvertizeParams,
+} from './echo-replicator';
+
+// TODO(dmaretskyi): Move out of @dxos/echo-pipeline.
+
+/**
+ * Used to replicate with other peers over the network.
+ */
+export class MeshEchoReplicator implements EchoReplicator {
+  private readonly _connections = new Set<MeshReplicatorConnection>();
+  /**
+   * Using automerge peerId as a key.
+   */
+  private readonly _connectionsPerPeer = new Map<string, MeshReplicatorConnection>();
+
+  /**
+   * spaceKey -> deviceKey[]
+   */
+  private readonly _authorizedDevices = new ComplexMap<PublicKey, ComplexSet<PublicKey>>(PublicKey.hash);
+
+  private _context: EchoReplicatorContext | null = null;
+
+  async connect(context: EchoReplicatorContext): Promise<void> {
+    this._context = context;
+  }
+
+  async disconnect() {
+    for (const connection of this._connections) {
+      await connection.close();
+    }
+    this._connections.clear();
+    this._connectionsPerPeer.clear();
+
+    this._context = null;
+  }
+
+  createExtension(): AutomergeReplicator {
+    invariant(this._context);
+
+    const connection: MeshReplicatorConnection = new MeshReplicatorConnection({
+      ownPeerId: this._context.peerId,
+      onRemoteConnected: async () => {
+        log('onRemoteConnected', { peerId: connection.peerId });
+        invariant(this._context);
+
+        if (!this._connectionsPerPeer.has(connection.peerId)) {
+          this._connectionsPerPeer.set(connection.peerId, connection);
+          await connection.enable();
+          this._context.onConnectionOpen(connection);
+        }
+      },
+      onRemoteDisconnected: async () => {
+        log('onRemoteDisconnected', { peerId: connection.peerId });
+        this._context?.onConnectionClosed(connection);
+        await connection.disable();
+        this._connectionsPerPeer.delete(connection.peerId);
+        this._connections.delete(connection);
+      },
+      shouldAdvertize: async (params: ShouldAdvertizeParams) => {
+        log('shouldAdvertize', { peerId: connection.peerId, documentId: params.documentId });
+        invariant(this._context);
+        try {
+          const spaceKey = await this._context.getContainingSpaceForDocument(params.documentId);
+          if (!spaceKey) {
+            log('space key not found for share policy check', {
+              peerId: connection.peerId,
+              documentId: params.documentId,
+            });
+            return false;
+          }
+
+          const authorizedDevices = this._authorizedDevices.get(spaceKey);
+
+          if (!connection.remoteDeviceKey) {
+            log('device key not found for share policy check', {
+              peerId: connection.peerId,
+              documentId: params.documentId,
+            });
+            return false;
+          }
+
+          const isAuthorized = authorizedDevices?.has(connection.remoteDeviceKey) ?? false;
+          log('share policy check', {
+            localPeer: this._context.peerId,
+            remotePeer: connection.peerId,
+            documentId: params.documentId,
+            deviceKey: connection.remoteDeviceKey,
+            spaceKey,
+            isAuthorized,
+          });
+          return isAuthorized;
+        } catch (err) {
+          log.catch(err);
+          return false;
+        }
+      },
+    });
+    this._connections.add(connection);
+
+    return connection.replicatorExtension;
+  }
+
+  authorizeDevice(spaceKey: PublicKey, deviceKey: PublicKey) {
+    log('authorizeDevice', { spaceKey, deviceKey });
+    defaultMap(this._authorizedDevices, spaceKey, () => new ComplexSet(PublicKey.hash)).add(deviceKey);
+  }
+}
+
+type MeshReplicatorConnectionParams = {
+  ownPeerId: string;
+  onRemoteConnected: () => Promise<void>;
+  onRemoteDisconnected: () => Promise<void>;
+  shouldAdvertize: (params: ShouldAdvertizeParams) => Promise<boolean>;
+};
+
+class MeshReplicatorConnection extends Resource implements ReplicatorConnection {
+  public readable: ReadableStream<Message>;
+  public writable: WritableStream<Message>;
+  public remoteDeviceKey: PublicKey | null = null;
+
+  public readonly replicatorExtension: AutomergeReplicator;
+
+  private _remotePeerId: string | null = null;
+  private _isEnabled = false;
+
+  constructor(private readonly _params: MeshReplicatorConnectionParams) {
+    super();
+
+    let readableStreamController!: ReadableStreamDefaultController<Message>;
+    this.readable = new ReadableStream<Message>({
+      start: (controller) => {
+        readableStreamController = controller;
+        this._ctx.onDispose(() => controller.close());
+      },
+    });
+
+    this.writable = new WritableStream<Message>({
+      write: async (message: Message, controller) => {
+        // TODO(dmaretskyi): Show we block on RPC completing here?
+        this.replicatorExtension.sendSyncMessage({ payload: cbor.encode(message) }).catch((err) => {
+          controller.error(err);
+        });
+      },
+    });
+
+    this.replicatorExtension = new AutomergeReplicator(
+      {
+        peerId: this._params.ownPeerId,
+      },
+      {
+        onStartReplication: async (info, remotePeerId /** Teleport ID */) => {
+          // Note: We store only one extension per peer.
+          //       There can be a case where two connected peers have more than one teleport connection between them
+          //       and each of them uses different teleport connections to send messages.
+          //       It works because we receive messages from all teleport connections and Automerge Repo dedup them.
+          // TODO(mykola): Use only one teleport connection per peer.
+
+          // TODO(dmaretskyi): Critical bug.
+          // - two peers get connected via swarm 1
+          // - they get connected via swarm 2
+          // - swarm 1 gets disconnected
+          // - automerge repo thinks that peer 2 got disconnected even though swarm 2 is still active
+
+          this.remoteDeviceKey = remotePeerId;
+
+          // Set automerge id.
+          this._remotePeerId = info.id;
+
+          log('onStartReplication', { id: info.id, thisPeerId: this.peerId, remotePeerId: remotePeerId.toHex() });
+
+          await this._params.onRemoteConnected();
+        },
+        onSyncMessage: async ({ payload }) => {
+          if (!this._isEnabled) {
+            return;
+          }
+          const message = cbor.decode(payload) as Message;
+          // Note: automerge Repo dedup messages.
+          readableStreamController.enqueue(message);
+        },
+        onClose: async () => {
+          if (!this._isEnabled) {
+            return;
+          }
+          await this._params.onRemoteDisconnected();
+        },
+      },
+    );
+  }
+
+  get peerId(): string {
+    invariant(this._remotePeerId != null, 'Remote peer has not connected yet.');
+    return this._remotePeerId;
+  }
+
+  async shouldAdvertize(params: ShouldAdvertizeParams): Promise<boolean> {
+    return this._params.shouldAdvertize(params);
+  }
+
+  /**
+   * Start exchanging messages with the remote peer.
+   * Call after the remote peer has connected.
+   */
+  async enable() {
+    invariant(this._remotePeerId != null, 'Remote peer has not connected yet.');
+    this._isEnabled = true;
+  }
+
+  /**
+   * Stop exchanging messages with the remote peer.
+   */
+  async disable() {
+    this._isEnabled = false;
+  }
+}

package/src/automerge/migrations.ts

@@ -4,14 +4,14 @@
 
 import { type StorageKey } from '@dxos/automerge/automerge-repo';
 import { IndexedDBStorageAdapter } from '@dxos/automerge/automerge-repo-storage-indexeddb';
-import { type SubLevelDB } from '@dxos/kv-store';
+import { type SublevelDB } from '@dxos/kv-store';
 import { log } from '@dxos/log';
 import { StorageType, type Directory } from '@dxos/random-access-storage';
 
 import { AutomergeStorageAdapter } from './automerge-storage-adapter';
 import { encodingOptions } from './leveldb-storage-adapter';
 
-export const levelMigration = async ({ db, directory }: { db: SubLevelDB; directory: Directory }) => {
+export const levelMigration = async ({ db, directory }: { db: SublevelDB; directory: Directory }) => {
   // Note: Make automigration from previous storage to leveldb here.
   const isNewLevel = !(await db
     .iterator<StorageKey, Uint8Array>({

package/src/db-host/snapshot-manager.ts

@@ -6,7 +6,7 @@ import { type Context, cancelWithContext } from '@dxos/context';
 import { PublicKey } from '@dxos/keys';
 import { schema } from '@dxos/protocols';
 import { BlobMeta } from '@dxos/protocols/proto/dxos/echo/blob';
-import { SpaceSnapshot } from '@dxos/protocols/proto/dxos/echo/snapshot';
+import { type SpaceSnapshot } from '@dxos/protocols/proto/dxos/echo/snapshot';
 import { type BlobStore, type BlobSync } from '@dxos/teleport-extension-object-sync';
 
 import { type SnapshotStore } from './snapshot-store';

package/src/db-host/snapshot-store.ts

@@ -5,7 +5,7 @@
 import { subtleCrypto } from '@dxos/crypto';
 import { schema } from '@dxos/protocols';
 import { type StoredSnapshotInfo } from '@dxos/protocols/proto/dxos/devtools/host';
-import { SpaceSnapshot } from '@dxos/protocols/proto/dxos/echo/snapshot';
+import { type SpaceSnapshot } from '@dxos/protocols/proto/dxos/echo/snapshot';
 import { type Directory } from '@dxos/random-access-storage';
 
 const SpaceSnapshot = schema.getCodecForType('dxos.echo.snapshot.SpaceSnapshot');

package/src/metadata/metadata-store.ts

@@ -14,11 +14,11 @@ import { DataCorruptionError, STORAGE_VERSION, schema } from '@dxos/protocols';
 import { Invitation, SpaceState } from '@dxos/protocols/proto/dxos/client/services';
 import {
   type ControlPipelineSnapshot,
-  EchoMetadata,
+  type EchoMetadata,
   type SpaceMetadata,
   type IdentityRecord,
   type SpaceCache,
-  LargeSpaceMetadata,
+  type LargeSpaceMetadata,
 } from '@dxos/protocols/proto/dxos/echo/metadata';
 import { type Directory, type File } from '@dxos/random-access-storage';
 import { type Timeframe } from '@dxos/timeframe';

package/src/space/space-manager.ts

@@ -3,7 +3,7 @@
 //
 
 import { synchronized, trackLeaks } from '@dxos/async';
-import { type DelegateInvitationCredential } from '@dxos/credentials';
+import { type DelegateInvitationCredential, type MemberInfo } from '@dxos/credentials';
 import { failUndefined } from '@dxos/debug';
 import { type FeedStore } from '@dxos/feed-store';
 import { PublicKey } from '@dxos/keys';
@@ -44,6 +44,7 @@ export type ConstructSpaceParams = {
   onAuthorizedConnection: (session: Teleport) => void;
   onAuthFailure?: (session: Teleport) => void;
   onDelegatedInvitationStatusChange: (invitation: DelegateInvitationCredential, isActive: boolean) => Promise<void>;
+  onMemberRolesChanged: (member: MemberInfo[]) => Promise<void>;
 };
 
 /**
@@ -87,6 +88,7 @@ export class SpaceManager {
     onAuthorizedConnection,
     onAuthFailure,
     onDelegatedInvitationStatusChange,
+    onMemberRolesChanged,
     memberKey,
   }: ConstructSpaceParams) {
     log.trace('dxos.echo.space-manager.construct-space', trace.begin({ id: this._instanceId }));
@@ -115,6 +117,7 @@ export class SpaceManager {
       snapshotManager,
       memberKey,
       onDelegatedInvitationStatusChange,
+      onMemberRolesChanged,
     });
     this._spaces.set(space.key, space);
 

package/src/space/space-protocol.ts

@@ -70,6 +70,12 @@ export class SpaceProtocol {
 
   private readonly _feeds = new Set<FeedWrapper<FeedMessage>>();
   private readonly _sessions = new ComplexMap<PublicKey, SpaceProtocolSession>(PublicKey.hash);
+  // TODO(burdon): Move to config (with sensible defaults).
+  private readonly _topology = new MMSTTopology({
+    originateConnections: 4,
+    maxPeers: 10,
+    sampleSize: 20,
+  });
 
   private _connection?: SwarmConnection;
 
@@ -117,13 +123,6 @@ export class SpaceProtocol {
     // TODO(burdon): Document why empty buffer.
     const credentials = await this._swarmIdentity.credentialProvider(Buffer.from(''));
 
-    // TODO(burdon): Move to config (with sensible defaults).
-    const topologyConfig = {
-      originateConnections: 4,
-      maxPeers: 10,
-      sampleSize: 20,
-    };
-
     await this.blobSync.open();
 
     log('starting...');
@@ -132,13 +131,17 @@ export class SpaceProtocol {
       protocolProvider: this._createProtocolProvider(credentials),
       peerId: this._swarmIdentity.peerKey,
       topic,
-      topology: new MMSTTopology(topologyConfig),
+      topology: this._topology,
       label: `swarm ${topic.truncate()} for space ${this._spaceKey.truncate()}`,
     });
 
     log('started');
   }
 
+  public updateTopology() {
+    this._topology.forceUpdate();
+  }
+
   async stop() {
     await this.blobSync.close();
 

package/src/space/space.ts

@@ -3,8 +3,8 @@
 //
 
 import { Event, Mutex, synchronized, trackLeaks } from '@dxos/async';
-import { Resource, type Context, LifecycleState } from '@dxos/context';
-import { type FeedInfo, type DelegateInvitationCredential } from '@dxos/credentials';
+import { type Context, LifecycleState, Resource } from '@dxos/context';
+import { type DelegateInvitationCredential, type FeedInfo, type MemberInfo } from '@dxos/credentials';
 import { type FeedOptions, type FeedWrapper } from '@dxos/feed-store';
 import { invariant } from '@dxos/invariant';
 import { type PublicKey } from '@dxos/keys';
@@ -13,7 +13,7 @@ import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { AdmittedFeed, type Credential } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { type Timeframe } from '@dxos/timeframe';
 import { trace } from '@dxos/tracing';
-import { Callback, type AsyncCallback } from '@dxos/util';
+import { type AsyncCallback, Callback } from '@dxos/util';
 
 import { ControlPipeline } from './control-pipeline';
 import { type SpaceProtocol } from './space-protocol';
@@ -37,6 +37,7 @@ export type SpaceParams = {
   snapshotId?: string | undefined;
 
   onDelegatedInvitationStatusChange: (invitation: DelegateInvitationCredential, isActive: boolean) => Promise<void>;
+  onMemberRolesChanged: (member: MemberInfo[]) => Promise<void>;
 };
 
 export type CreatePipelineParams = {
@@ -109,6 +110,10 @@ export class Space extends Resource {
       log('onDelegatedInvitationRemoved', { invitation });
       await params.onDelegatedInvitationStatusChange(invitation, false);
     });
+    this._controlPipeline.onMemberRoleChanged.set(async (changedMembers) => {
+      log('onMemberRoleChanged', () => ({ changedMembers: changedMembers.map((m) => [m.key, m.role]) }));
+      await params.onMemberRolesChanged(changedMembers);
+    });
 
     // Start replicating the genesis feed.
     this.protocol = params.protocol;

package/src/testing/test-agent-builder.ts

@@ -191,6 +191,7 @@ export class TestAgent {
         );
       },
       onDelegatedInvitationStatusChange: async () => {},
+      onMemberRolesChanged: async () => {},
     });
     await space.setControlFeed(controlFeed);
     await space.setDataFeed(dataFeed);