@dxos/echo-pipeline 0.8.3 → 0.8.4-main.05e74ebcff

package/src/automerge/automerge-subduction.test.ts

@@ -0,0 +1,330 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { Repo, generateAutomergeUrl, initSubduction, parseAutomergeUrl } from '@automerge/automerge-repo';
+import {
+  AuthenticatedTransport,
+  BlobMeta,
+  CommitId,
+  Digest,
+  MemorySigner,
+  MemoryStorage,
+  Nonce,
+  SedimentreeId,
+  Subduction,
+  commitIdOfBase58Id,
+} from '@automerge/automerge-subduction';
+import { beforeAll, describe, test } from 'vitest';
+
+import { sleep } from '@dxos/async';
+
+class AsyncQueue<T> {
+  private _items: T[] = [];
+  private _waiters: ((item: T) => void)[] = [];
+
+  push(item: T): void {
+    const waiter = this._waiters.shift();
+    if (waiter) {
+      waiter(item);
+    } else {
+      this._items.push(item);
+    }
+  }
+
+  async pull(): Promise<T> {
+    const item = this._items.shift();
+    if (item !== undefined) {
+      return item;
+    }
+    return new Promise<T>((resolve) => this._waiters.push(resolve));
+  }
+
+  clear(): void {
+    this._items.length = 0;
+    this._waiters.length = 0;
+  }
+}
+
+/**
+ * Pure-JS transport backed by async queues.
+ * Implements the subduction Transport interface for in-memory testing.
+ */
+class MemoryTransport {
+  private _outBytes: AsyncQueue<Uint8Array>;
+  private _inBytes: AsyncQueue<Uint8Array>;
+  private _disconnectCallbacks: (() => void)[] = [];
+
+  constructor(outBytes: AsyncQueue<Uint8Array>, inBytes: AsyncQueue<Uint8Array>) {
+    this._outBytes = outBytes;
+    this._inBytes = inBytes;
+  }
+
+  async sendBytes(bytes: Uint8Array): Promise<void> {
+    this._outBytes.push(new Uint8Array(bytes));
+  }
+
+  async recvBytes(): Promise<Uint8Array> {
+    return this._inBytes.pull();
+  }
+
+  async disconnect(): Promise<void> {
+    for (const callback of this._disconnectCallbacks) {
+      callback();
+    }
+    this._inBytes.clear();
+    this._outBytes.clear();
+  }
+
+  onDisconnect(callback: () => void): void {
+    this._disconnectCallbacks.push(callback);
+  }
+}
+
+const createMemoryTransportPair = (): [MemoryTransport, MemoryTransport] => {
+  const bytesAB = new AsyncQueue<Uint8Array>();
+  const bytesBA = new AsyncQueue<Uint8Array>();
+  return [new MemoryTransport(bytesAB, bytesBA), new MemoryTransport(bytesBA, bytesAB)];
+};
+
+/**
+ * Build a deterministic {@link CommitId} seeded by a small integer. Values are arbitrary;
+ * used in tests only to satisfy the `head` parameter of `Subduction.addCommit`.
+ */
+const commitIdOf = (seed: number): CommitId => CommitId.fromBytes(new Uint8Array(32).fill(seed));
+
+describe('automerge-subduction', () => {
+  beforeAll(async () => {
+    await initSubduction();
+  });
+
+  test('creates signer and signs payloads', async ({ expect }) => {
+    const signer = MemorySigner.generate();
+    const signature = await signer.sign(new Uint8Array([1, 2, 3]));
+    const verifyingKey = signer.verifyingKey();
+    const peerId = signer.peerId();
+
+    expect(signature).toBeInstanceOf(Uint8Array);
+    expect(signature.length).toBe(64);
+    expect(verifyingKey).toBeInstanceOf(Uint8Array);
+    expect(verifyingKey.length).toBe(32);
+    expect(peerId.toString().length).toBeGreaterThan(0);
+  });
+
+  test('round-trips digest bytes and hex', ({ expect }) => {
+    const digestBytes = new Uint8Array(32).fill(2);
+    const digest = Digest.fromBytes(digestBytes);
+    const fromHex = Digest.fromHexString(digest.toHexString());
+
+    expect(digest.toBytes()).toEqual(digestBytes);
+    expect(digest.toHexString()).toHaveLength(64);
+    expect(fromHex.toBytes()).toEqual(digestBytes);
+  });
+
+  test('nonce random bytes are fixed-width', ({ expect }) => {
+    const nonce = Nonce.random();
+    expect(nonce.bytes.length).toBe(8);
+  });
+
+  test('blob meta reports digest and size', ({ expect }) => {
+    const blobMeta = new BlobMeta(new Uint8Array([5, 6, 7, 8]));
+    expect(blobMeta.sizeBytes).toBe(4n);
+    expect(blobMeta.digest().toBytes().length).toBe(32);
+  });
+
+  test('stores and loads sedimentree IDs in memory storage', async ({ expect }) => {
+    const storage = new MemoryStorage();
+    const id = SedimentreeId.fromBytes(new Uint8Array(32).fill(7));
+
+    await storage.saveSedimentreeId(id);
+    const ids = await storage.loadAllSedimentreeIds();
+
+    expect(ids).toHaveLength(1);
+    expect(ids[0].toBytes()).toEqual(id.toBytes());
+  });
+
+  test('adds a commit and exposes it through subduction read APIs', async ({ expect }) => {
+    const signer = MemorySigner.generate();
+    const storage = new MemoryStorage();
+    const subduction = new Subduction(signer, storage);
+    const id = SedimentreeId.fromBytes(new Uint8Array(32).fill(9));
+    const blob = new Uint8Array([10, 11, 12]);
+
+    const requestedFragment = await subduction.addCommit(id, commitIdOf(1), [], blob);
+    const commits = await subduction.getCommits(id);
+    const blobs = await subduction.getBlobs(id);
+
+    expect(requestedFragment).toBeUndefined();
+    expect(commits).toBeDefined();
+    expect(commits).toHaveLength(1);
+    expect(blobs).toHaveLength(1);
+    expect(blobs[0]).toEqual(blob);
+  });
+
+  test('throws on invalid base58 input', ({ expect }) => {
+    expect(() => commitIdOfBase58Id('contains-0')).toThrow();
+  });
+
+  test('automerge-repo URL helpers round-trip document IDs', ({ expect }) => {
+    const url = generateAutomergeUrl();
+    const parsed = parseAutomergeUrl(url);
+
+    expect(parsed.documentId.length).toBeGreaterThan(0);
+    expect(`automerge:${parsed.documentId}`).toBe(url);
+  });
+
+  test('automerge-repo creates documents with subduction signer', async ({ expect }) => {
+    const signer = MemorySigner.generate();
+    const repo = new Repo({
+      network: [],
+      signer,
+    });
+
+    const handle = repo.create<{ title?: string }>();
+    handle.change((doc) => {
+      doc.title = 'hello';
+    });
+
+    expect(handle.doc()?.title).toBe('hello');
+    await repo.shutdown();
+  });
+
+  test('establishes subduction connection and syncs via AuthenticatedTransport', async ({ expect }) => {
+    const signerA = MemorySigner.generate();
+    const signerB = MemorySigner.generate();
+    const subductionA = new Subduction(signerA, new MemoryStorage());
+    const subductionB = new Subduction(signerB, new MemoryStorage());
+
+    const [transportA, transportB] = createMemoryTransportPair();
+
+    const [authA, authB] = await Promise.all([
+      AuthenticatedTransport.setup(transportA, signerA, signerB.peerId()),
+      AuthenticatedTransport.accept(transportB, signerB),
+    ]);
+
+    await subductionA.addConnection(authA);
+    await subductionB.addConnection(authB);
+
+    const peersA = await subductionA.getConnectedPeerIds();
+    const peersB = await subductionB.getConnectedPeerIds();
+    expect(peersA).toHaveLength(1);
+    expect(peersB).toHaveLength(1);
+
+    const sid = SedimentreeId.fromBytes(new Uint8Array(32).fill(42));
+    await subductionA.addCommit(sid, commitIdOf(2), [], new Uint8Array([1, 2, 3]));
+
+    const result = await subductionA.syncWithAllPeers(sid, true);
+    expect(result.entries().length).toBeGreaterThan(0);
+
+    const blobsOnB = await subductionB.getBlobs(sid);
+    expect(blobsOnB).toHaveLength(1);
+  }, 10_000);
+
+  test('syncs between two subduction instances using connectTransport', async ({ expect }) => {
+    const signerA = MemorySigner.generate();
+    const signerB = MemorySigner.generate();
+    const subductionA = new Subduction(signerA, new MemoryStorage(), 'test-service');
+    const subductionB = new Subduction(signerB, new MemoryStorage(), 'test-service');
+
+    const [transportA, transportB] = createMemoryTransportPair();
+
+    const [peerIdB, peerIdA] = await Promise.all([
+      subductionA.connectTransport(transportA, 'test-service'),
+      subductionB.acceptTransport(transportB, 'test-service'),
+    ]);
+
+    expect(peerIdB.toString()).toBe(signerB.peerId().toString());
+    expect(peerIdA.toString()).toBe(signerA.peerId().toString());
+
+    const sid = SedimentreeId.fromBytes(new Uint8Array(32).fill(99));
+    await subductionA.addCommit(sid, commitIdOf(3), [], new Uint8Array([4, 5, 6]));
+
+    await subductionA.syncWithAllPeers(sid, false);
+
+    const blobsOnB = await subductionB.getBlobs(sid);
+    expect(blobsOnB).toHaveLength(1);
+    expect(blobsOnB[0]).toEqual(new Uint8Array([4, 5, 6]));
+  }, 10_000);
+
+  test('full sync exchanges all sedimentrees between peers', async ({ expect }) => {
+    const signerA = MemorySigner.generate();
+    const signerB = MemorySigner.generate();
+    const subductionA = new Subduction(signerA, new MemoryStorage());
+    const subductionB = new Subduction(signerB, new MemoryStorage());
+
+    // Add commits on both sides before connecting.
+    const sidA = SedimentreeId.fromBytes(new Uint8Array(32).fill(1));
+    const sidB = SedimentreeId.fromBytes(new Uint8Array(32).fill(2));
+    await subductionA.addCommit(sidA, commitIdOf(5), [], new Uint8Array([10, 20]));
+    await subductionB.addCommit(sidB, commitIdOf(6), [], new Uint8Array([30, 40]));
+
+    const [transportA, transportB] = createMemoryTransportPair();
+    const [authA, authB] = await Promise.all([
+      AuthenticatedTransport.setup(transportA, signerA, signerB.peerId()),
+      AuthenticatedTransport.accept(transportB, signerB),
+    ]);
+    await subductionA.addConnection(authA);
+    await subductionB.addConnection(authB);
+
+    // Full sync exchanges everything.
+    await subductionA.fullSyncWithPeer(signerB.peerId());
+    await subductionB.fullSyncWithPeer(signerA.peerId());
+
+    const blobsAonB = await subductionB.getBlobs(sidA);
+    const blobsBonA = await subductionA.getBlobs(sidB);
+    expect(blobsAonB).toHaveLength(1);
+    expect(blobsAonB[0]).toEqual(new Uint8Array([10, 20]));
+    expect(blobsBonA).toHaveLength(1);
+    expect(blobsBonA[0]).toEqual(new Uint8Array([30, 40]));
+  }, 10_000);
+
+  // After B drops handshake state and rehydrates from durable storage, does
+  // subduction re-handshake on the reused logical channel when A pushes?
+  // Empirical answer: no — A's auto-broadcast does not reach B'.
+  test(
+    'does not auto-rehydrate handshake on a logical connection where one end dropped it',
+    { timeout: 15_000 },
+    async ({ expect }) => {
+      const signerA = MemorySigner.generate();
+      const signerB = MemorySigner.generate();
+      const storageA = new MemoryStorage();
+      const storageB = new MemoryStorage();
+      const [transportA, transportB] = createMemoryTransportPair();
+
+      const subA = new Subduction(signerA, storageA, 'svc');
+      let subB = new Subduction(signerB, storageB, 'svc');
+
+      const [authA, authB] = await Promise.all([
+        AuthenticatedTransport.setup(transportA, signerA, signerB.peerId()),
+        AuthenticatedTransport.accept(transportB, signerB),
+      ]);
+      await subA.addConnection(authA);
+      await subB.addConnection(authB);
+
+      const sid = SedimentreeId.fromBytes(new Uint8Array(32).fill(7));
+      await subA.addCommit(sid, commitIdOf(1), [], new Uint8Array([1, 2, 3]));
+      await expect.poll(() => subB.getBlobs(sid).then((bs) => bs.length), { timeout: 5_000 }).toEqual(1);
+
+      // Simulate B crash: clearing the queue's waiters halts the orphaned
+      // wasm pump that survives `free()`. Do NOT use `disconnectAll` /
+      // `disconnectFromPeer` — those send a graceful goodbye A would observe.
+      await transportB.disconnect();
+      subB.free();
+
+      subB = await Subduction.hydrate(signerB, storageB, 'svc');
+      expect(await subB.getBlobs(sid)).toHaveLength(1);
+      expect(await subB.getConnectedPeerIds()).toHaveLength(0);
+      expect(await subA.getConnectedPeerIds()).toHaveLength(1);
+
+      await subA.addCommit(sid, commitIdOf(2), [commitIdOf(1)], new Uint8Array([4, 5, 6]));
+      await sleep(500);
+
+      // TODO(mykola): When subduction-core grows channel-level handshake
+      // recovery, flip to `.toHaveLength(2)` and drop "does not" from name.
+      expect(await subB.getBlobs(sid)).toHaveLength(1);
+
+      subA.free();
+      subB.free();
+    },
+  );
+});

package/src/automerge/collection-synchronizer.test.ts

@@ -2,15 +2,17 @@
 // Copyright 2024 DXOS.org
 //
 
-import type { PeerId } from '@automerge/automerge-repo';
-import { onTestFinished, describe, expect, test } from 'vitest';
+import * as A from '@automerge/automerge';
+import type { DocumentId, PeerId } from '@automerge/automerge-repo';
+import { describe, onTestFinished, test } from 'vitest';
 
 import { sleep } from '@dxos/async';
+import { range } from '@dxos/util';
 
-import { CollectionSynchronizer, diffCollectionState, type CollectionState } from './collection-synchronizer';
+import { type CollectionState, CollectionSynchronizer, diffCollectionState } from './collection-synchronizer';
 
 describe('CollectionSynchronizer', () => {
-  test('sync two peers', async () => {
+  test('sync two peers', async ({ expect }) => {
     const LATENCY = 10;
 
     const peerId1 = 'peer1' as PeerId;
@@ -53,22 +55,99 @@ describe('CollectionSynchronizer', () => {
     peer1.onConnectionOpen(peerId2);
     peer2.onConnectionOpen(peerId1);
 
+    const updated = Promise.all([
+      peer1.remoteStateUpdated.waitFor((ev) => ev.collectionId === collectionId && ev.peerId === peerId2),
+      peer2.remoteStateUpdated.waitFor((ev) => ev.collectionId === collectionId && ev.peerId === peerId1),
+    ]);
+
     peer1.setLocalCollectionState(collectionId, STATE_1);
     peer2.setLocalCollectionState(collectionId, STATE_2);
 
     peer1.refreshCollection(collectionId);
     peer2.refreshCollection(collectionId);
 
-    await Promise.all([
-      peer1.remoteStateUpdated.waitFor((ev) => ev.collectionId === collectionId && ev.peerId === peerId2),
-      peer2.remoteStateUpdated.waitFor((ev) => ev.collectionId === collectionId && ev.peerId === peerId1),
-    ]);
+    await updated;
 
     expect(peer1.getRemoteCollectionStates(collectionId).get(peerId2)).to.deep.equal(STATE_2);
     expect(peer2.getRemoteCollectionStates(collectionId).get(peerId1)).to.deep.equal(STATE_1);
   });
 
-  test('diff collection state', () => {
+  test('pushes state to all interested peers on setLocalCollectionState', async ({ expect }) => {
+    // Push covers peers that queried us before we had state — `onCollectionStateQueried`
+    // silently drops that case; without this push the asker would wait up to POLL_INTERVAL
+    // for its polling loop to retry (further gated by MIN_QUERY_INTERVAL).
+    const peerId1 = 'peer1' as PeerId;
+    const peerId2 = 'peer2' as PeerId;
+    const collectionId = 'collection-test';
+
+    const sentStates: Array<{ peerId: PeerId; state: CollectionState }> = [];
+    const peer = await new CollectionSynchronizer({
+      queryCollectionState: () => {},
+      sendCollectionState: (_collectionId, peerId, state) => {
+        sentStates.push({ peerId, state });
+      },
+      shouldSyncCollection: () => true,
+    }).open();
+    onTestFinished(async () => {
+      await peer.close();
+    });
+
+    peer.onConnectionOpen(peerId1);
+    peer.onConnectionOpen(peerId2);
+
+    peer.setLocalCollectionState(collectionId, STATE_1);
+    await sleep(10);
+
+    expect(sentStates.map((m) => m.peerId).sort()).to.deep.equal([peerId1, peerId2].sort());
+    expect(sentStates.every((m) => m.state === STATE_1)).to.equal(true);
+  });
+
+  test('does not send to a peer after onConnectionClosed', async ({ expect }) => {
+    // Regression: `onConnectionClosed` must remove the peer from `interestedPeers`,
+    // otherwise the next `_broadcastLocalState` calls `sendCollectionState` for a
+    // now-stale peerId and the real network adapter throws "Connection not found.".
+    const peerId1 = 'peer1' as PeerId;
+    const peerId2 = 'peer2' as PeerId;
+    const collectionId = 'collection-test';
+
+    const connected = new Set<PeerId>([peerId1, peerId2]);
+    const sentTo: PeerId[] = [];
+    const peer = await new CollectionSynchronizer({
+      queryCollectionState: () => {},
+      sendCollectionState: (_collectionId, peerId) => {
+        if (!connected.has(peerId)) {
+          throw new Error('Connection not found.');
+        }
+        sentTo.push(peerId);
+      },
+      shouldSyncCollection: () => true,
+    }).open();
+    onTestFinished(async () => {
+      await peer.close();
+    });
+
+    // Activate the collection before any peer connects so the connect-time
+    // microtask adds peers to `interestedPeers` without ever calling
+    // `_broadcastLocalState` (so `lastBroadcast` stays unset for them).
+    peer.setLocalCollectionState(collectionId, STATE_1);
+    await sleep(10);
+
+    peer.onConnectionOpen(peerId1);
+    peer.onConnectionOpen(peerId2);
+    await sleep(10);
+
+    connected.delete(peerId2);
+    peer.onConnectionClosed(peerId2);
+
+    // Without the fix, peer2 is still in `interestedPeers` with no `lastBroadcast`
+    // entry, so the broadcast gate passes and `sendCollectionState` throws.
+    peer.setLocalCollectionState(collectionId, STATE_2);
+    await sleep(10);
+
+    expect(sentTo).to.deep.equal([peerId1]);
+  });
+
+  test('diff collection state', ({ expect }) => {
     const diff = diffCollectionState(STATE_1, STATE_2);
 
     expect(diff).to.deep.equal({
@@ -79,18 +158,20 @@ describe('CollectionSynchronizer', () => {
   });
 });
 
+const TEST_HEADS = range(4).map((i) => A.getHeads(A.from({ i: i.toString() })));
+
 const STATE_1: CollectionState = {
   documents: {
-    a: ['1'],
-    b: ['2'],
-    c: ['3'],
-  },
+    a: TEST_HEADS[0],
+    b: TEST_HEADS[1],
+    c: TEST_HEADS[2],
+  } as Record<DocumentId, A.Heads>,
 };
 
 const STATE_2: CollectionState = {
   documents: {
-    a: ['1'],
-    b: ['4'],
-    d: ['3'],
-  },
+    a: TEST_HEADS[0],
+    b: TEST_HEADS[3],
+    d: TEST_HEADS[2],
+  } as Record<DocumentId, A.Heads>,
 };