@dxos/echo-pipeline 0.3.11-main.d7d4c52 → 0.3.11-main.d8b8a39

package/src/automerge/automerge-host.test.ts

@@ -2,16 +2,19 @@
 // Copyright 2023 DXOS.org
 //
 
+import { randomBytes } from 'crypto';
 import expect from 'expect';
+import waitForExpect from 'wait-for-expect';
 
-import { Trigger, asyncTimeout, sleep } from '@dxos/async';
+import { asyncTimeout, sleep } from '@dxos/async';
 import { type Message, NetworkAdapter, type PeerId, Repo } from '@dxos/automerge/automerge-repo';
 import { invariant } from '@dxos/invariant';
-import { log } from '@dxos/log';
 import { StorageType, createStorage } from '@dxos/random-access-storage';
-import { describe, test } from '@dxos/test';
+import { TestBuilder as TeleportBuilder, TestPeer as TeleportPeer } from '@dxos/teleport/testing';
+import { afterTest, describe, test } from '@dxos/test';
+import { arrayToBuffer, bufferToArray } from '@dxos/util';
 
-import { AutomergeHost } from './automerge-host';
+import { AutomergeHost, AutomergeStorageAdapter, MeshNetworkAdapter } from './automerge-host';
 
 describe('AutomergeHost', () => {
   test('can create documents', () => {
@@ -44,13 +47,12 @@ describe('AutomergeHost', () => {
   });
 
   test('basic networking', async () => {
-    type Context = { client: Trigger<TestAdapter>; host: Trigger<TestAdapter> };
-    const context: Context = {
-      client: new Trigger<TestAdapter>(),
-      host: new Trigger<TestAdapter>(),
-    };
-    const hostAdapter = new TestAdapter(context, 'host');
-    const clientAdapter = new TestAdapter(context, 'client');
+    const hostAdapter: TestAdapter = new TestAdapter({
+      send: (message: Message) => clientAdapter.receive(message),
+    });
+    const clientAdapter: TestAdapter = new TestAdapter({
+      send: (message: Message) => hostAdapter.receive(message),
+    });
 
     const host = new Repo({
       network: [hostAdapter],
@@ -60,6 +62,8 @@ describe('AutomergeHost', () => {
     });
     hostAdapter.ready();
     clientAdapter.ready();
+    hostAdapter.peerCandidate(clientAdapter.peerId!);
+    clientAdapter.peerCandidate(hostAdapter.peerId!);
 
     const handle = host.create();
     const text = 'Hello world';
@@ -68,17 +72,197 @@ describe('AutomergeHost', () => {
     });
 
     const docOnClient = client.find(handle.url);
-    await asyncTimeout(docOnClient.whenReady(), 3_000);
-    expect(docOnClient.docSync().text).toEqual(text);
+    expect((await asyncTimeout(docOnClient.doc(), 1000)).text).toEqual(text);
   });
 
-  test('doubled connection', async () => {});
-});
+  test('recovering from a lost connection', async () => {
+    let connectionState: 'on' | 'off' = 'on';
+
+    const hostAdapter: TestAdapter = new TestAdapter({
+      send: (message: Message) => connectionState === 'on' && sleep(10).then(() => clientAdapter.receive(message)),
+    });
+    const clientAdapter: TestAdapter = new TestAdapter({
+      send: (message: Message) => connectionState === 'on' && sleep(10).then(() => hostAdapter.receive(message)),
+    });
+
+    const host = new Repo({
+      network: [hostAdapter],
+    });
+    const client = new Repo({
+      network: [clientAdapter],
+    });
+
+    // Establish connection.
+    hostAdapter.ready();
+    clientAdapter.ready();
+    hostAdapter.peerCandidate(clientAdapter.peerId!);
+    clientAdapter.peerCandidate(hostAdapter.peerId!);
+
+    const handle = host.create();
+    const docOnClient = client.find(handle.url);
+    {
+      const sanityText = 'Hello world';
+      handle.change((doc: any) => {
+        doc.sanityText = sanityText;
+      });
+
+      expect((await asyncTimeout(docOnClient.doc(), 1000)).sanityText).toEqual(sanityText);
+    }
+
+    // Disrupt connection.
+    const offlineText = 'This has been written while the connection was off';
+    {
+      connectionState = 'off';
+
+      handle.change((doc: any) => {
+        doc.offlineText = offlineText;
+      });
+
+      await sleep(100);
+      expect((await asyncTimeout(docOnClient.doc(), 1000)).offlineText).toBeUndefined();
+    }
+
+    // Re-establish connection.
+    const onlineText = 'This has been written after the connection was re-established';
+    {
+      connectionState = 'on';
+      hostAdapter.peerDisconnected(clientAdapter.peerId!);
+      clientAdapter.peerDisconnected(hostAdapter.peerId!);
+      hostAdapter.peerCandidate(clientAdapter.peerId!);
+      clientAdapter.peerCandidate(hostAdapter.peerId!);
+
+      handle.change((doc: any) => {
+        doc.onlineText = onlineText;
+      });
+      await sleep(100);
+      expect((await asyncTimeout(docOnClient.doc(), 1000)).onlineText).toEqual(onlineText);
+      expect((await asyncTimeout(docOnClient.doc(), 1000)).offlineText).toEqual(offlineText);
+    }
+  });
 
-type Context = { client: Trigger<TestAdapter>; host: Trigger<TestAdapter> };
+  test('integration test with teleport', async () => {
+    const createAutomergeRepo = () => {
+      const meshAdapter = new MeshNetworkAdapter();
+      const repo = new Repo({
+        network: [meshAdapter],
+      });
+      meshAdapter.ready();
+      return { repo, meshAdapter };
+    };
+    const peer1 = createAutomergeRepo();
+    const peer2 = createAutomergeRepo();
+    const handle = peer1.repo.create();
+
+    const teleportBuilder = new TeleportBuilder();
+    afterTest(() => teleportBuilder.destroy());
+
+    const [teleportPeer1, teleportPeer2] = teleportBuilder.createPeers({ factory: () => new TeleportPeer() });
+    {
+      // Initiate connection.
+      const [connection1, connection2] = await teleportBuilder.connect(teleportPeer1, teleportPeer2);
+      connection1.teleport.addExtension('automerge', peer1.meshAdapter.createExtension());
+      connection2.teleport.addExtension('automerge', peer2.meshAdapter.createExtension());
+
+      // Test connection.
+      const text = 'Hello world';
+      handle.change((doc: any) => {
+        doc.text = text;
+      });
+      const docOnPeer2 = peer2.repo.find(handle.url);
+      await waitForExpect(async () => expect((await asyncTimeout(docOnPeer2.doc(), 1000)).text).toEqual(text), 1000);
+    }
+
+    const offlineText = 'This has been written while the connection was off';
+    {
+      // Disconnect peers.
+      await teleportBuilder.disconnect(teleportPeer1, teleportPeer2);
+
+      // Make offline changes.
+      const offlineText = 'This has been written while the connection was off';
+      handle.change((doc: any) => {
+        doc.offlineText = offlineText;
+      });
+      const docOnPeer2 = peer2.repo.find(handle.url);
+      await sleep(100);
+      expect((await asyncTimeout(docOnPeer2.doc(), 1000)).offlineText).toBeUndefined();
+    }
+
+    {
+      // Reconnect peers.
+      const [connection1, connection2] = await teleportBuilder.connect(teleportPeer1, teleportPeer2);
+      connection1.teleport.addExtension('automerge', peer1.meshAdapter.createExtension());
+      connection2.teleport.addExtension('automerge', peer2.meshAdapter.createExtension());
+
+      // Wait for offline changes to be synced.
+      const docOnPeer2 = peer2.repo.find(handle.url);
+      await waitForExpect(
+        async () => expect((await asyncTimeout(docOnPeer2.doc(), 1000)).offlineText).toEqual(offlineText),
+        1000,
+      );
+
+      // Test connection.
+      const onlineText = 'This has been written after the connection was re-established';
+      handle.change((doc: any) => {
+        doc.onlineText = onlineText;
+      });
+      await waitForExpect(
+        async () => expect((await asyncTimeout(docOnPeer2.doc(), 1000)).onlineText).toEqual(onlineText),
+        1000,
+      );
+    }
+  });
+
+  describe('storage', () => {
+    test('load range on node', async () => {
+      const root = `/tmp/${randomBytes(16).toString('hex')}`;
+      {
+        const storage = createStorage({ type: StorageType.NODE, root });
+        const adapter = new AutomergeStorageAdapter(storage.createDirectory());
+
+        await adapter.save(['test', '1'], bufferToArray(Buffer.from('one')));
+        await adapter.save(['test', '2'], bufferToArray(Buffer.from('two')));
+        await adapter.save(['bar', '1'], bufferToArray(Buffer.from('bar')));
+      }
+
+      {
+        const storage = createStorage({ type: StorageType.NODE, root });
+        const adapter = new AutomergeStorageAdapter(storage.createDirectory());
+
+        const range = await adapter.loadRange(['test']);
+        expect(range.map((chunk) => arrayToBuffer(chunk.data!).toString())).toEqual(['one', 'two']);
+        expect(range.map((chunk) => chunk.key)).toEqual([
+          ['test', '1'],
+          ['test', '2'],
+        ]);
+      }
+    });
+
+    test('removeRange on node', async () => {
+      const root = `/tmp/${randomBytes(16).toString('hex')}`;
+      {
+        const storage = createStorage({ type: StorageType.NODE, root });
+        const adapter = new AutomergeStorageAdapter(storage.createDirectory());
+        await adapter.save(['test', '1'], bufferToArray(Buffer.from('one')));
+        await adapter.save(['test', '2'], bufferToArray(Buffer.from('two')));
+        await adapter.save(['bar', '1'], bufferToArray(Buffer.from('bar')));
+      }
+
+      {
+        const storage = createStorage({ type: StorageType.NODE, root });
+        const adapter = new AutomergeStorageAdapter(storage.createDirectory());
+        await adapter.removeRange(['test']);
+        const range = await adapter.loadRange(['test']);
+        expect(range.map((chunk) => arrayToBuffer(chunk.data!).toString())).toEqual([]);
+        const range2 = await adapter.loadRange(['bar']);
+        expect(range2.map((chunk) => arrayToBuffer(chunk.data!).toString())).toEqual(['bar']);
+        expect(range2.map((chunk) => chunk.key)).toEqual([['bar', '1']]);
+      }
+    });
+  });
+});
 
 class TestAdapter extends NetworkAdapter {
-  constructor(public readonly context: Context, public readonly role: 'host' | 'client') {
+  constructor(private readonly _params: { send: (message: Message) => void }) {
     super();
   }
 
@@ -90,32 +274,24 @@ class TestAdapter extends NetworkAdapter {
 
   override connect(peerId: PeerId) {
     this.peerId = peerId;
-    this.context[this.role].wake(this);
-    this.context[this.role === 'host' ? 'client' : 'host']
-      .wait()
-      .then((adapter) => {
-        invariant(adapter.peerId, 'Peer id is not set');
-        this.emit('peer-candidate', { peerId: adapter.peerId });
-      })
-      .catch((error) => {
-        log.catch(error);
-      });
+  }
+
+  peerCandidate(peerId: PeerId) {
+    invariant(peerId, 'PeerId is required');
+    this.emit('peer-candidate', { peerId });
+  }
+
+  peerDisconnected(peerId: PeerId) {
+    invariant(peerId, 'PeerId is required');
+    this.emit('peer-disconnected', { peerId });
   }
 
   override send(message: Message) {
-    this.context[this.role === 'host' ? 'client' : 'host']
-      .wait()
-      .then((adapter) => {
-        adapter.receive(message);
-      })
-      .catch((error) => {
-        log.catch(error);
-      });
+    this._params.send(message);
   }
 
   override disconnect() {
     this.peerId = undefined;
-    this.context[this.role].reset();
   }
 
   receive(message: Message) {

package/src/automerge/automerge-host.ts

@@ -12,12 +12,13 @@ import {
   type StorageKey,
   cbor,
 } from '@dxos/automerge/automerge-repo';
+import { IndexedDBStorageAdapter } from '@dxos/automerge/automerge-repo-storage-indexeddb';
 import { Stream } from '@dxos/codec-protobuf';
 import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 import { type HostInfo, type SyncRepoRequest, type SyncRepoResponse } from '@dxos/protocols/proto/dxos/echo/service';
 import { type PeerInfo } from '@dxos/protocols/proto/dxos/mesh/teleport/automerge';
-import { type Directory } from '@dxos/random-access-storage';
+import { StorageType, type Directory } from '@dxos/random-access-storage';
 import { AutomergeReplicator } from '@dxos/teleport-extension-automerge-replicator';
 import { arrayToBuffer, bufferToArray } from '@dxos/util';
 
@@ -25,12 +26,17 @@ export class AutomergeHost {
   private readonly _repo: Repo;
   private readonly _meshNetwork: MeshNetworkAdapter;
   private readonly _clientNetwork: LocalHostNetworkAdapter;
-  private readonly _storage: AutomergeStorageAdapter;
+  private readonly _storage: StorageAdapter;
 
   constructor(storageDirectory: Directory) {
     this._meshNetwork = new MeshNetworkAdapter();
     this._clientNetwork = new LocalHostNetworkAdapter();
-    this._storage = new AutomergeStorageAdapter(storageDirectory);
+
+    // TODO(mykola): Delete specific handling of IDB storage.
+    this._storage =
+      storageDirectory.type === StorageType.IDB
+        ? new IndexedDBStorageAdapter(storageDirectory.path, 'data')
+        : new AutomergeStorageAdapter(storageDirectory);
     this._repo = new Repo({
       network: [this._clientNetwork, this._meshNetwork],
       storage: this._storage,
@@ -166,7 +172,7 @@ class LocalHostNetworkAdapter extends NetworkAdapter {
 /**
  * Used to replicate with other peers over the network.
  */
-class MeshNetworkAdapter extends NetworkAdapter {
+export class MeshNetworkAdapter extends NetworkAdapter {
   private readonly _extensions: Map<string, AutomergeReplicator> = new Map();
 
   /**
@@ -227,10 +233,13 @@ class MeshNetworkAdapter extends NetworkAdapter {
           this.emit('message', message);
         },
         onClose: async () => {
-          peerInfo &&
-            this.emit('peer-disconnected', {
-              peerId: peerInfo.id as PeerId,
-            });
+          if (!peerInfo) {
+            return;
+          }
+          this.emit('peer-disconnected', {
+            peerId: peerInfo.id as PeerId,
+          });
+          this._extensions.delete(peerInfo.id);
         },
       },
     );
@@ -238,7 +247,7 @@ class MeshNetworkAdapter extends NetworkAdapter {
   }
 }
 
-class AutomergeStorageAdapter extends StorageAdapter {
+export class AutomergeStorageAdapter extends StorageAdapter {
   constructor(private readonly _directory: Directory) {
     super();
   }
@@ -267,7 +276,7 @@ class AutomergeStorageAdapter extends StorageAdapter {
     // TODO(dmaretskyi): Better deletion.
     const filename = this._getFilename(key);
     const file = this._directory.getOrCreateFile(filename);
-    await file.truncate?.(0);
+    await file.destroy();
   }
 
   override async loadRange(keyPrefix: StorageKey): Promise<Chunk[]> {
@@ -295,8 +304,8 @@ class AutomergeStorageAdapter extends StorageAdapter {
       entries
         .filter((entry) => entry.startsWith(filename))
         .map(async (entry): Promise<void> => {
-          const file = this._directory.getOrCreateFile(filename);
-          await file.truncate?.(0);
+          const file = this._directory.getOrCreateFile(entry);
+          await file.destroy();
         }),
     );
   }

package/src/automerge/index.ts

@@ -2,4 +2,4 @@
 // Copyright 2023 DXOS.org
 //
 
-export * from './automerge-host';
+export { AutomergeHost } from './automerge-host';

package/src/metadata/metadata-store.ts

@@ -48,7 +48,14 @@ export class MetadataStore {
 
   public readonly update = new Event<EchoMetadata>();
 
-  constructor(private readonly _directory: Directory) {}
+  /**
+   * @internal
+   */
+  readonly _directory: Directory;
+
+  constructor(directory: Directory) {
+    this._directory = directory;
+  }
 
   get metadata(): EchoMetadata {
     return this._metadata;
@@ -94,7 +101,10 @@ export class MetadataStore {
     }
   }
 
-  private async _writeFile<T>(file: File, codec: Codec<T>, data: T): Promise<void> {
+  /**
+   * @internal
+   */
+  async _writeFile<T>(file: File, codec: Codec<T>, data: T): Promise<void> {
     const encoded = arrayToBuffer(codec.encode(data));
     const checksum = CRC32.buf(encoded);
 

package/src/testing/change-metadata.ts

@@ -0,0 +1,27 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import { log } from '@dxos/log';
+import { schema } from '@dxos/protocols';
+import type { Storage } from '@dxos/random-access-storage';
+
+import { MetadataStore } from '../metadata';
+
+const EchoMetadata = schema.getCodecForType('dxos.echo.metadata.EchoMetadata');
+
+/**
+ * This function will change the storage version in the metadata.
+ * This will break your storage and make it unusable.
+ * Use this only for testing purposes.
+ */
+export const changeStorageVersionInMetadata = async (storage: Storage, version: number) => {
+  log.info('Changing storage version in metadata. USE ONLY FOR TESTING.');
+  const metadata = new MetadataStore(storage.createDirectory('metadata'));
+  await metadata.load();
+  const echoMetadata = metadata.metadata;
+  echoMetadata.version = version;
+  const file = metadata._directory.getOrCreateFile('EchoMetadata');
+  await metadata._writeFile(file, EchoMetadata, echoMetadata);
+  await metadata._directory.flush();
+};

package/src/testing/index.ts

@@ -2,6 +2,7 @@
 // Copyright 2022 DXOS.org
 //
 
+export * from './change-metadata';
 export * from './test-agent-builder';
 export * from './test-feed-builder';
 export * from './util';