@dxos/echo-pipeline 0.8.3 → 0.8.4-main.1068cf700f

package/src/automerge/automerge-repo.test.ts

@@ -3,6 +3,7 @@
 //
 
 import {
+  next as A,
   type Heads,
   change,
   clone,
@@ -10,7 +11,6 @@ import {
   from,
   getBackend,
   getHeads,
-  next as A,
   save,
   saveSince,
 } from '@automerge/automerge';
@@ -19,28 +19,31 @@ import {
   type DocHandle,
   type DocumentId,
   type HandleState,
-  type StorageAdapterInterface,
+  type Message,
   type PeerId,
   Repo,
   type SharePolicy,
+  type StorageAdapterInterface,
   generateAutomergeUrl,
   parseAutomergeUrl,
 } from '@automerge/automerge-repo';
-import { onTestFinished, describe, expect, test } from 'vitest';
+import { describe, expect, onTestFinished, test } from 'vitest';
 
 import { asyncTimeout, sleep } from '@dxos/async';
 import { randomBytes } from '@dxos/crypto';
 import { PublicKey } from '@dxos/keys';
 import { createTestLevel } from '@dxos/kv-store/testing';
+import { log } from '@dxos/log';
 import { TestBuilder as TeleportBuilder, TestPeer as TeleportPeer } from '@dxos/teleport/testing';
 import { openAndClose } from '@dxos/test-utils';
 import { isNonNullable, range } from '@dxos/util';
 
+import { TestAdapter, type TestConnectionStateProvider } from '../testing';
+
 import { FIND_PARAMS } from './automerge-host';
 import { EchoNetworkAdapter } from './echo-network-adapter';
 import { LevelDBStorageAdapter } from './leveldb-storage-adapter';
 import { MeshEchoReplicator } from './mesh-echo-replicator';
-import { TestAdapter, type TestConnectionStateProvider } from '../testing';
 
 const HOST_AND_CLIENT: [string, string] = ['host', 'client'];
 
@@ -161,32 +164,6 @@ describe('AutomergeRepo', () => {
       expect((await asyncTimeout(client.find<any>(handle.url), 1000))!.doc().text).to.equal(text);
     });
 
-    test('share policy gets enabled afterwards', async () => {
-      let sharePolicy = false;
-
-      const { repos, adapters } = await createHostClientRepoTopology({
-        sharePolicy: async () => sharePolicy,
-      });
-      const [host, client] = repos;
-      await connectAdapters(adapters);
-
-      const handle = host.create();
-      const text = 'Hello world';
-      handle.change((doc: any) => {
-        doc.text = text;
-      });
-
-      {
-        await client.find(handle.url, { allowableStates: ['unavailable'] });
-      }
-
-      sharePolicy = true;
-
-      {
-        await client.find(handle.url, { allowableStates: ['unavailable'] });
-      }
-    });
-
     test('two documents and share policy switching', async () => {
       const allowedDocs: DocumentId[] = [];
 
@@ -420,13 +397,7 @@ describe('AutomergeRepo', () => {
 
     test('client creates doc and syncs with a Repo', async () => {
       const repo = new Repo({ network: [] });
-      const receiveByServer = async (blob: Uint8Array, docId: DocumentId) => {
-        const serverHandle = await repo.find(docId, FIND_PARAMS);
-        serverHandle.update((doc) => {
-          return A.loadIncremental(doc, blob);
-        });
-      };
-
+      const receiveByServer = (blob: Uint8Array, docId: DocumentId) => repo.import<any>(blob, { docId });
       let clientDoc = A.from<{ field?: string }>({});
       const { documentId } = parseAutomergeUrl(generateAutomergeUrl());
       // Sync handshake.
@@ -434,7 +405,7 @@ describe('AutomergeRepo', () => {
 
       // Sync protocol.
       const sendDoc = async (doc: A.Doc<any>) => {
-        await receiveByServer(saveSince(doc, sentHeads), documentId);
+        receiveByServer(saveSince(doc, sentHeads), documentId);
         sentHeads = getHeads(doc);
       };
 
@@ -456,10 +427,9 @@ describe('AutomergeRepo', () => {
 
       const repo = new Repo({ network: [], storage });
       const receiveByServer = async (blob: Uint8Array, docId: DocumentId) => {
-        const serverHandle = await repo.find(docId, FIND_PARAMS);
-        serverHandle.update((doc) => {
-          return A.loadIncremental(doc, blob);
-        });
+        repo.import<any>(blob, { docId });
+        // TODO(mykola): This should not be required. Document is not persisted without it.
+        await repo.flush([docId]);
       };
 
       let clientDoc = A.from<{ field?: string }>({ field: 'foo' });
@@ -516,6 +486,139 @@ describe('AutomergeRepo', () => {
       await asyncTimeout(handleB.whenReady(), 1000);
       expect(handleB.doc()!.text).to.equal(text);
     });
+
+    test('retry share config', async () => {
+      let announce = false;
+
+      const { repos, adapters } = await createRepoTopology({
+        peers: ['A', 'B'],
+        connections: [['A', 'B']],
+        options: {
+          shareConfig: {
+            access: async () => true,
+            announce: async () => announce,
+          },
+        },
+        onMessage: (message) => {
+          console.log(`${message.senderId} -> ${message.targetId}: ${message.type} ${message.documentId ?? ''}`);
+        },
+      });
+      const [repoA, repoB] = repos;
+      await connectAdapters(adapters);
+      console.log({ peersA: repoA.peers, peersB: repoB.peers });
+
+      const docA = repoA.create({ text: 'Hello world' });
+
+      const docB = await repoB.find(docA.url, { allowableStates: ['ready', 'unavailable'] });
+      expect(docB.state).to.equal('unavailable');
+      log.info('unavailable');
+
+      announce = true;
+      repoB.shareConfigChanged();
+      await docB.whenReady();
+    });
+  });
+
+  describe('create2 tests', () => {
+    test('basic document creation', async () => {
+      const repo = new Repo({ network: [] });
+
+      const handle = await repo.create2<{ text: string }>();
+      expect(handle.doc()).to.deep.equal({});
+
+      handle.change((doc: any) => {
+        doc.text = 'Hello World';
+      });
+
+      expect(handle.doc()?.text).to.equal('Hello World');
+    });
+
+    test('document mutation', async () => {
+      const repo = new Repo({ network: [] });
+
+      const handle = await repo.create2<{ counter: number }>({ counter: 0 });
+
+      for (let i = 1; i <= 10; i++) {
+        handle.change((doc: any) => {
+          doc.counter = i;
+        });
+      }
+
+      expect(handle.doc()?.counter).to.equal(10);
+    });
+
+    test('import document and mutate', async () => {
+      const repo = new Repo({ network: [] });
+
+      const original = await repo.create2<{ text: string }>({ text: 'original' });
+      const blob = A.save(original.doc()!);
+
+      const imported = repo.import<{ text: string }>(blob);
+      expect(imported.doc()?.text).to.equal('original');
+
+      imported.change((doc: any) => {
+        doc.text = 'mutated';
+      });
+      expect(imported.doc()?.text).to.equal('mutated');
+    });
+
+    test('reload document with flush', async () => {
+      const path = createTmpPath();
+      const text = 'Hello World!';
+      let url: AutomergeUrl;
+
+      {
+        const level = createTestLevel(path);
+        const storage = await createLevelAdapter(level);
+        const repo = new Repo({ network: [], storage });
+        const handle = await repo.create2<{ text: string }>();
+        url = handle.url;
+        handle.change((doc: any) => {
+          doc.text = text;
+        });
+        await repo.flush([handle.documentId]);
+        await level.close();
+      }
+
+      {
+        const level = createTestLevel(path);
+        const storage = await createLevelAdapter(level);
+        const repo = new Repo({ network: [], storage });
+        const handle = await repo.find<{ text: string }>(url);
+        await handle.whenReady();
+        expect(handle.doc()?.text).to.equal(text);
+        await level.close();
+      }
+    });
+
+    test('reload document without flush', async () => {
+      const path = createTmpPath();
+      const text = 'Hello World!';
+      let url: AutomergeUrl;
+
+      {
+        const level = createTestLevel(path);
+        const storage = await createLevelAdapter(level);
+        const repo = new Repo({ network: [], storage });
+        const handle = await repo.create2<{ text: string }>();
+        url = handle.url;
+        handle.change((doc: any) => {
+          doc.text = text;
+        });
+        // No explicit flush - rely on auto-save.
+        await sleep(200);
+        await level.close();
+      }
+
+      {
+        const level = createTestLevel(path);
+        const storage = await createLevelAdapter(level);
+        const repo = new Repo({ network: [], storage });
+        const handle = await repo.find<{ text: string }>(url);
+        await handle.whenReady();
+        expect(handle.doc()?.text).to.equal(text);
+      }
+    });
   });
 
   describe('teleport', () => {
@@ -677,10 +780,66 @@ describe('AutomergeRepo', () => {
       await doc.whenReady();
       expect(doc.doc()).to.deep.eq(document.doc());
     });
+
+    test('document is passively replicated to connected peers', async () => {
+      const [spaceKey] = PublicKey.randomSequence();
+
+      const teleportBuilder = new TeleportBuilder();
+      onTestFinished(() => teleportBuilder.destroy());
+
+      const peer1 = await createTeleportTestPeer(teleportBuilder, spaceKey);
+      const peer2 = await createTeleportTestPeer(teleportBuilder, spaceKey);
+
+      const handle = peer1.repo.create();
+      handle.change((doc: any) => (doc.text = 'hello'));
+      await connectPeers(spaceKey, teleportBuilder, peer1, peer2);
+
+      await expect
+        .poll(async () => {
+          const doc = peer2.repo.handles[handle.documentId];
+          await doc.whenReady();
+          return doc.doc()!.text;
+        })
+        .toEqual('hello');
+    });
+
+    test('imported document is passively replicated to connected peers', async () => {
+      let blob: Uint8Array;
+      let documentId: DocumentId;
+      {
+        const repo = new Repo();
+        const handle = repo.create();
+        handle.change((doc: any) => (doc.text = 'hello'));
+        blob = A.save(handle.doc()!);
+        documentId = handle.documentId;
+      }
+
+      const [spaceKey] = PublicKey.randomSequence();
+
+      const teleportBuilder = new TeleportBuilder();
+      onTestFinished(() => teleportBuilder.destroy());
+
+      const peer1 = await createTeleportTestPeer(teleportBuilder, spaceKey);
+      const peer2 = await createTeleportTestPeer(teleportBuilder, spaceKey);
+      await connectPeers(spaceKey, teleportBuilder, peer1, peer2);
+
+      const handle = peer1.repo.import(blob, { docId: documentId });
+      await handle.whenReady();
+
+      await expect
+        .poll(
+          async () => {
+            const doc = peer2.repo.handles[handle.documentId];
+            await doc.whenReady();
+            return doc.doc()!.text;
+          },
+          { timeout: 1_000 },
+        )
+        .toEqual('hello');
+    });
   });
 
-  const createLevelAdapter = async () => {
-    const level = createTestLevel();
+  const createLevelAdapter = async (level = createTestLevel()) => {
     const storage = new LevelDBStorageAdapter({ db: level.sublevel('automerge') });
     await openAndClose(level, storage);
     return storage;
@@ -702,19 +861,23 @@ describe('AutomergeRepo', () => {
   };
 });
 
+type ShareConfig = Exclude<ConstructorParameters<typeof Repo>[0], undefined>['shareConfig'];
+
 type ConnectedRepoOptions = {
   storages?: StorageAdapterInterface[];
   connectionStateProvider?: TestConnectionStateProvider;
   sharePolicy?: SharePolicy;
+  shareConfig?: ShareConfig;
 };
 
 const createRepoTopology = async <Peers extends string[], Peer extends string = Peers[number]>(args: {
   peers: Peers;
   connections: [Peer, Peer][];
   options?: ConnectedRepoOptions;
+  onMessage?: (message: Message) => void;
 }) => {
   const adapters = args.connections.map(
-    () => TestAdapter.createPair(args.options?.connectionStateProvider) as [TestAdapter, TestAdapter],
+    () => TestAdapter.createPair(args.options?.connectionStateProvider, args.onMessage) as [TestAdapter, TestAdapter],
   );
   const repos = args.peers.map((peerId, peerIndex) => {
     const network = adapters
@@ -730,7 +893,8 @@ const createRepoTopology = async <Peers extends string[], Peer extends string =
       peerId: peerId as PeerId,
       storage: args.options?.storages?.[peerIndex],
       network,
-      sharePolicy: args.options?.sharePolicy ?? (async () => true),
+      sharePolicy: (args.options?.sharePolicy ?? args.options?.shareConfig) ? undefined : async () => true,
+      shareConfig: args.options?.shareConfig,
     });
   });
   return { repos, adapters };
@@ -816,3 +980,7 @@ const connectPeers = async (
 };
 
 type TeleportTestPeer = { repo: Repo; meshAdapter: MeshEchoReplicator; teleport: TeleportPeer };
+
+export const createTmpPath = (): string => {
+  return `/tmp/dxos-${PublicKey.random().toHex()}`;
+};

package/src/automerge/collection-synchronizer.test.ts

@@ -2,12 +2,15 @@
 // Copyright 2024 DXOS.org
 //
 
-import type { PeerId } from '@automerge/automerge-repo';
-import { onTestFinished, describe, expect, test } from 'vitest';
+import { type Heads } from '@automerge/automerge';
+import * as Automerge from '@automerge/automerge';
+import type { DocumentId, PeerId } from '@automerge/automerge-repo';
+import { describe, expect, onTestFinished, test } from 'vitest';
 
 import { sleep } from '@dxos/async';
+import { range } from '@dxos/util';
 
-import { CollectionSynchronizer, diffCollectionState, type CollectionState } from './collection-synchronizer';
+import { type CollectionState, CollectionSynchronizer, diffCollectionState } from './collection-synchronizer';
 
 describe('CollectionSynchronizer', () => {
   test('sync two peers', async () => {
@@ -53,16 +56,18 @@ describe('CollectionSynchronizer', () => {
     peer1.onConnectionOpen(peerId2);
     peer2.onConnectionOpen(peerId1);
 
+    const updated = Promise.all([
+      peer1.remoteStateUpdated.waitFor((ev) => ev.collectionId === collectionId && ev.peerId === peerId2),
+      peer2.remoteStateUpdated.waitFor((ev) => ev.collectionId === collectionId && ev.peerId === peerId1),
+    ]);
+
     peer1.setLocalCollectionState(collectionId, STATE_1);
     peer2.setLocalCollectionState(collectionId, STATE_2);
 
     peer1.refreshCollection(collectionId);
     peer2.refreshCollection(collectionId);
 
-    await Promise.all([
-      peer1.remoteStateUpdated.waitFor((ev) => ev.collectionId === collectionId && ev.peerId === peerId2),
-      peer2.remoteStateUpdated.waitFor((ev) => ev.collectionId === collectionId && ev.peerId === peerId1),
-    ]);
+    await updated;
 
     expect(peer1.getRemoteCollectionStates(collectionId).get(peerId2)).to.deep.equal(STATE_2);
     expect(peer2.getRemoteCollectionStates(collectionId).get(peerId1)).to.deep.equal(STATE_1);
@@ -79,18 +84,20 @@ describe('CollectionSynchronizer', () => {
   });
 });
 
+const TEST_HEADS = range(4).map((i) => Automerge.getHeads(Automerge.from({ i: i.toString() })));
+
 const STATE_1: CollectionState = {
   documents: {
-    a: ['1'],
-    b: ['2'],
-    c: ['3'],
-  },
+    a: TEST_HEADS[0],
+    b: TEST_HEADS[1],
+    c: TEST_HEADS[2],
+  } as Record<DocumentId, Heads>,
 };
 
 const STATE_2: CollectionState = {
   documents: {
-    a: ['1'],
-    b: ['4'],
-    d: ['3'],
-  },
+    a: TEST_HEADS[0],
+    b: TEST_HEADS[3],
+    d: TEST_HEADS[2],
+  } as Record<DocumentId, Heads>,
 };

package/src/automerge/collection-synchronizer.ts

@@ -2,20 +2,22 @@
 // Copyright 2024 DXOS.org
 //
 
-import { next as am } from '@automerge/automerge';
+import { next as A, type Heads } from '@automerge/automerge';
 import type { DocumentId, PeerId } from '@automerge/automerge-repo';
+import * as Array from 'effect/Array';
+import * as Record from 'effect/Record';
 
-import { asyncReturn, Event, scheduleTask, scheduleTaskInterval } from '@dxos/async';
-import { Resource, type Context } from '@dxos/context';
+import { Event, asyncReturn, scheduleTask, scheduleTaskInterval } from '@dxos/async';
+import { type Context, Resource } from '@dxos/context';
 import { log } from '@dxos/log';
 import { trace } from '@dxos/tracing';
 import { defaultMap } from '@dxos/util';
 
 const MIN_QUERY_INTERVAL = 5_000;
 
-const POLL_INTERVAL = 30_000;
+const POLL_INTERVAL = 10_000;
 
-export type CollectionSynchronizerParams = {
+export type CollectionSynchronizerProps = {
   sendCollectionState: (collectionId: string, peerId: PeerId, state: CollectionState) => void;
   queryCollectionState: (collectionId: string, peerId: PeerId) => void;
   shouldSyncCollection: (collectionId: string, peerId: PeerId) => boolean;
@@ -26,9 +28,9 @@ export type CollectionSynchronizerParams = {
  */
 @trace.resource()
 export class CollectionSynchronizer extends Resource {
-  private readonly _sendCollectionState: CollectionSynchronizerParams['sendCollectionState'];
-  private readonly _queryCollectionState: CollectionSynchronizerParams['queryCollectionState'];
-  private readonly _shouldSyncCollection: CollectionSynchronizerParams['shouldSyncCollection'];
+  private readonly _sendCollectionState: CollectionSynchronizerProps['sendCollectionState'];
+  private readonly _queryCollectionState: CollectionSynchronizerProps['queryCollectionState'];
+  private readonly _shouldSyncCollection: CollectionSynchronizerProps['shouldSyncCollection'];
 
   /**
    * CollectionId -> State.
@@ -40,7 +42,7 @@ export class CollectionSynchronizer extends Resource {
 
   public readonly remoteStateUpdated = new Event<{ collectionId: string; peerId: PeerId; newDocsAppeared: boolean }>();
 
-  constructor(params: CollectionSynchronizerParams) {
+  constructor(params: CollectionSynchronizerProps) {
     super();
     this._sendCollectionState = params.sendCollectionState;
     this._queryCollectionState = params.queryCollectionState;
@@ -76,6 +78,10 @@ export class CollectionSynchronizer extends Resource {
     log('setLocalCollectionState', { collectionId, state });
     this._getOrCreatePerCollectionState(collectionId).localState = state;
 
+    for (const peerId of this._connectedPeers) {
+      this._diffCollectionState(collectionId, peerId);
+    }
+
     queueMicrotask(async () => {
       if (!this._ctx.disposed && this._activeCollections.has(collectionId)) {
         this._refreshInterestedPeers(collectionId);
@@ -117,6 +123,7 @@ export class CollectionSynchronizer extends Resource {
    * Callback when a connection to a peer is established.
    */
   onConnectionOpen(peerId: PeerId): void {
+    log('onConnectionOpen', { peerId });
     const spanId = getSpanName(peerId);
     trace.spanStart({
       id: spanId,
@@ -146,6 +153,8 @@ export class CollectionSynchronizer extends Resource {
    * Callback when a connection to a peer is closed.
    */
   onConnectionClosed(peerId: PeerId): void {
+    log('onConnectionClosed', { peerId });
+
     this._connectedPeers.delete(peerId);
 
     for (const perCollectionState of this._perCollectionStates.values()) {
@@ -171,8 +180,20 @@ export class CollectionSynchronizer extends Resource {
     log('onRemoteStateReceived', { collectionId, peerId, state });
     validateCollectionState(state);
     const perCollectionState = this._getOrCreatePerCollectionState(collectionId);
-    const existingState = perCollectionState.remoteStates.get(peerId) ?? { documents: {} };
-    const diff = diffCollectionState(existingState, state);
+    perCollectionState.remoteStates.set(peerId, state);
+    this._diffCollectionState(collectionId, peerId);
+  }
+
+  private _diffCollectionState(collectionId: string, peerId: PeerId) {
+    const perCollectionState = this._getOrCreatePerCollectionState(collectionId);
+    const remoteState = perCollectionState.remoteStates.get(peerId);
+    if (!remoteState) {
+      return;
+    }
+
+    log('diffCollectionState', { collectionId, peerId });
+    const localState = perCollectionState.localState ?? { documents: {} };
+    const diff = diffCollectionState(localState, remoteState);
     const spanId = getSpanName(peerId);
     if (diff.different.length === 0) {
       trace.spanEnd(spanId);
@@ -186,9 +207,20 @@ export class CollectionSynchronizer extends Resource {
         attributes: { peerId },
       });
     }
-    if (diff.missingOnLocal.length > 0 || diff.different.length > 0) {
-      perCollectionState.remoteStates.set(peerId, state);
-      this.remoteStateUpdated.emit({ peerId, collectionId, newDocsAppeared: diff.missingOnLocal.length > 0 });
+    log('diff', {
+      localState: localState.documents,
+      remoteState: remoteState.documents,
+      missingOnLocal: diff.missingOnLocal,
+      missingOnRemote: diff.missingOnRemote,
+      different: diff.different,
+    });
+    if (diff.missingOnLocal.length > 0 || diff.different.length > 0 || diff.missingOnRemote.length > 0) {
+      log('emit remote state update');
+      this.remoteStateUpdated.emit({
+        peerId,
+        collectionId,
+        newDocsAppeared: diff.missingOnLocal.length > 0,
+      });
     }
   }
 
@@ -223,7 +255,7 @@ export type CollectionState = {
   /**
    * DocumentId -> Heads.
    */
-  documents: Record<string, string[]>;
+  documents: Record<DocumentId, Heads>;
 };
 
 export type CollectionStateDiff = {
@@ -233,18 +265,21 @@ export type CollectionStateDiff = {
 };
 
 export const diffCollectionState = (local: CollectionState, remote: CollectionState): CollectionStateDiff => {
-  const allDocuments = new Set<DocumentId>([...Object.keys(local.documents), ...Object.keys(remote.documents)] as any);
+  const localDocuments = Record.filter(local.documents, (heads) => heads.length > 0);
+  const remoteDocuments = Record.filter(remote.documents, (heads) => heads.length > 0);
+  // NOTE: Using `Array.union` is slow.
+  const allDocuments = [...new Set([...Record.keys(localDocuments), ...Record.keys(remoteDocuments)])] as DocumentId[];
 
   const missingOnRemote: DocumentId[] = [];
   const missingOnLocal: DocumentId[] = [];
   const different: DocumentId[] = [];
   for (const documentId of allDocuments) {
-    if (!local.documents[documentId]) {
-      missingOnLocal.push(documentId as DocumentId);
-    } else if (!remote.documents[documentId]) {
-      missingOnRemote.push(documentId as DocumentId);
-    } else if (!am.equals(local.documents[documentId], remote.documents[documentId])) {
-      different.push(documentId as DocumentId);
+    if (!localDocuments[documentId]) {
+      missingOnLocal.push(documentId);
+    } else if (!remoteDocuments[documentId]) {
+      missingOnRemote.push(documentId);
+    } else if (!A.equals(local.documents[documentId], remote.documents[documentId])) {
+      different.push(documentId);
     }
   }
 

package/src/automerge/echo-data-monitor.ts

@@ -5,7 +5,7 @@
 import { type Message } from '@automerge/automerge-repo';
 
 import { type TimeAware, trace } from '@dxos/tracing';
-import { CircularBuffer, mapValues, SlidingWindowSummary, type SlidingWindowSummaryConfig } from '@dxos/util';
+import { CircularBuffer, SlidingWindowSummary, type SlidingWindowSummaryConfig, mapValues } from '@dxos/util';
 
 import { type NetworkDataMonitor } from './echo-network-adapter';
 import { type StorageAdapterDataMonitor } from './leveldb-storage-adapter';

package/src/automerge/echo-network-adapter.test.ts

@@ -2,10 +2,10 @@
 // Copyright 2024 DXOS.org
 //
 
-import { cbor, type PeerId } from '@automerge/automerge-repo';
-import { onTestFinished, describe, expect, test } from 'vitest';
+import { type PeerId, cbor } from '@automerge/automerge-repo';
+import { describe, expect, onTestFinished, test } from 'vitest';
 
-import { sleep, Trigger, waitForCondition } from '@dxos/async';
+import { Trigger, sleep, waitForCondition } from '@dxos/async';
 import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { type SyncMessage } from '@dxos/protocols/proto/dxos/mesh/teleport/automerge';