@dxos/echo-pipeline 0.8.3 → 0.8.4-main.1c7ec43d41

package/package.json

@@ -1,38 +1,35 @@
 {
   "name": "@dxos/echo-pipeline",
-  "version": "0.8.3",
+  "version": "0.8.4-main.1c7ec43d41",
   "description": "ECHO database.",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dxos/dxos"
+  },
   "license": "MIT",
   "author": "info@dxos.org",
-  "sideEffects": true,
+  "sideEffects": false,
   "type": "module",
   "exports": {
     ".": {
+      "source": "./src/index.ts",
       "types": "./dist/types/src/index.d.ts",
-      "browser": "./dist/lib/browser/index.mjs",
-      "node": "./dist/lib/node-esm/index.mjs"
+      "default": "./dist/lib/neutral/index.mjs"
     },
     "./testing": {
+      "source": "./src/testing/index.ts",
       "types": "./dist/types/src/testing/index.d.ts",
-      "browser": "./dist/lib/browser/testing/index.mjs",
-      "node": "./dist/lib/node-esm/testing/index.mjs"
+      "default": "./dist/lib/neutral/testing/index.mjs"
     },
     "./filter": {
+      "source": "./src/filter/index.ts",
       "types": "./dist/types/src/filter/index.d.ts",
-      "browser": "./dist/lib/browser/filter/index.mjs",
-      "node": "./dist/lib/node-esm/filter/index.mjs"
+      "default": "./dist/lib/neutral/filter/index.mjs"
     }
   },
   "types": "dist/types/src/index.d.ts",
-  "typesVersions": {
-    "*": {
-      "testing": [
-        "dist/types/src/testing/index.d.ts"
-      ]
-    }
-  },
   "files": [
     "testing.d.ts",
     "testing.js",
@@ -40,51 +37,56 @@
     "src"
   ],
   "dependencies": {
-    "@automerge/automerge": "3.0.0-beta.4",
-    "@automerge/automerge-repo": "2.0.1",
+    "@automerge/automerge": "3.2.6",
+    "@automerge/automerge-repo": "2.6.0-subduction.17",
+    "@automerge/automerge-subduction": "0.11.0",
+    "@effect/sql": "0.49.0",
+    "@effect/sql-sqlite-node": "0.50.1",
+    "bs58check": "^4.0.0",
     "crc-32": "^1.2.2",
-    "effect": "3.14.21",
+    "effect": "3.20.0",
+    "fast-deep-equal": "^3.1.3",
     "level-transcoder": "^1.0.1",
-    "lodash.isequal": "^4.5.0",
-    "@dxos/codec-protobuf": "0.8.3",
-    "@dxos/credentials": "0.8.3",
-    "@dxos/context": "0.8.3",
-    "@dxos/async": "0.8.3",
-    "@dxos/crypto": "0.8.3",
-    "@dxos/debug": "0.8.3",
-    "@dxos/echo": "0.8.3",
-    "@dxos/echo-protocol": "0.8.3",
-    "@dxos/edge-client": "0.8.3",
-    "@dxos/echo-schema": "0.8.3",
-    "@dxos/feed-store": "0.8.3",
-    "@dxos/errors": "0.8.3",
-    "@dxos/indexing": "0.8.3",
-    "@dxos/hypercore": "0.8.3",
-    "@dxos/invariant": "0.8.3",
-    "@dxos/keyring": "0.8.3",
-    "@dxos/keys": "0.8.3",
-    "@dxos/kv-store": "0.8.3",
-    "@dxos/log": "0.8.3",
-    "@dxos/messaging": "0.8.3",
-    "@dxos/network-manager": "0.8.3",
-    "@dxos/node-std": "0.8.3",
-    "@dxos/teleport": "0.8.3",
-    "@dxos/protocols": "0.8.3",
-    "@dxos/random-access-storage": "0.8.3",
-    "@dxos/teleport-extension-automerge-replicator": "0.8.3",
-    "@dxos/teleport-extension-gossip": "0.8.3",
-    "@dxos/teleport-extension-object-sync": "0.8.3",
-    "@dxos/teleport-extension-replicator": "0.8.3",
-    "@dxos/timeframe": "0.8.3",
-    "@dxos/tracing": "0.8.3",
-    "@dxos/util": "0.8.3",
-    "@dxos/typings": "0.8.3"
+    "@dxos/async": "0.8.4-main.1c7ec43d41",
+    "@dxos/codec-protobuf": "0.8.4-main.1c7ec43d41",
+    "@dxos/credentials": "0.8.4-main.1c7ec43d41",
+    "@dxos/context": "0.8.4-main.1c7ec43d41",
+    "@dxos/crypto": "0.8.4-main.1c7ec43d41",
+    "@dxos/debug": "0.8.4-main.1c7ec43d41",
+    "@dxos/echo": "0.8.4-main.1c7ec43d41",
+    "@dxos/echo-protocol": "0.8.4-main.1c7ec43d41",
+    "@dxos/edge-client": "0.8.4-main.1c7ec43d41",
+    "@dxos/errors": "0.8.4-main.1c7ec43d41",
+    "@dxos/effect": "0.8.4-main.1c7ec43d41",
+    "@dxos/feed": "0.8.4-main.1c7ec43d41",
+    "@dxos/feed-store": "0.8.4-main.1c7ec43d41",
+    "@dxos/hypercore": "0.8.4-main.1c7ec43d41",
+    "@dxos/index-core": "0.8.4-main.1c7ec43d41",
+    "@dxos/invariant": "0.8.4-main.1c7ec43d41",
+    "@dxos/keyring": "0.8.4-main.1c7ec43d41",
+    "@dxos/kv-store": "0.8.4-main.1c7ec43d41",
+    "@dxos/keys": "0.8.4-main.1c7ec43d41",
+    "@dxos/messaging": "0.8.4-main.1c7ec43d41",
+    "@dxos/network-manager": "0.8.4-main.1c7ec43d41",
+    "@dxos/node-std": "0.8.4-main.1c7ec43d41",
+    "@dxos/log": "0.8.4-main.1c7ec43d41",
+    "@dxos/protocols": "0.8.4-main.1c7ec43d41",
+    "@dxos/random-access-storage": "0.8.4-main.1c7ec43d41",
+    "@dxos/sql-sqlite": "0.8.4-main.1c7ec43d41",
+    "@dxos/teleport": "0.8.4-main.1c7ec43d41",
+    "@dxos/teleport-extension-automerge-replicator": "0.8.4-main.1c7ec43d41",
+    "@dxos/teleport-extension-gossip": "0.8.4-main.1c7ec43d41",
+    "@dxos/teleport-extension-object-sync": "0.8.4-main.1c7ec43d41",
+    "@dxos/timeframe": "0.8.4-main.1c7ec43d41",
+    "@dxos/teleport-extension-replicator": "0.8.4-main.1c7ec43d41",
+    "@dxos/tracing": "0.8.4-main.1c7ec43d41",
+    "@dxos/typings": "0.8.4-main.1c7ec43d41",
+    "@dxos/util": "0.8.4-main.1c7ec43d41"
   },
   "devDependencies": {
-    "@types/lodash.isequal": "^4.5.0",
     "fast-check": "^3.19.0",
     "get-port-please": "^3.1.1",
-    "@dxos/test-utils": "0.8.3"
+    "@dxos/test-utils": "0.8.4-main.1c7ec43d41"
   },
   "publishConfig": {
     "access": "public"

package/src/automerge/automerge-host-subduction.test.ts

@@ -0,0 +1,247 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { getHeads } from '@automerge/automerge';
+import * as Automerge from '@automerge/automerge';
+import { type DocumentId, type Heads, generateAutomergeUrl, parseAutomergeUrl } from '@automerge/automerge-repo';
+import { describe, onTestFinished, test } from 'vitest';
+
+import { sleep } from '@dxos/async';
+import { Context } from '@dxos/context';
+import { PublicKey } from '@dxos/keys';
+import type { LevelDB } from '@dxos/kv-store';
+import { createTestLevel } from '@dxos/kv-store/testing';
+import { openAndClose } from '@dxos/test-utils';
+import { range } from '@dxos/util';
+
+import { TestReplicationNetwork } from '../testing';
+import { AutomergeHost } from './automerge-host';
+
+describe('AutomergeHost with Subduction', () => {
+  test('can create documents', async ({ expect }) => {
+    const level = await createLevel();
+    const host = await setupAutomergeHost({ level });
+    const handle = await host.createDoc<any>();
+    handle.change((doc: any) => {
+      doc.text = 'Hello world';
+    });
+    await host.flush(Context.default());
+    expect(handle.doc()!.text).toEqual('Hello world');
+  });
+
+  test('changes are preserved in storage', async ({ expect }) => {
+    const level = await createLevel();
+    const host = await setupAutomergeHost({ level });
+    const handle = await host.createDoc<any>();
+    handle.change((doc: any) => {
+      doc.text = 'Hello world';
+    });
+    const url = handle.url;
+
+    await host.flush(Context.default());
+    await host.close();
+
+    const host2 = await setupAutomergeHost({ level });
+    const handle2 = await host2.loadDoc<any>(Context.default(), url);
+    await handle2.whenReady();
+    expect(handle2.doc()!.text).toEqual('Hello world');
+    await host2.flush(Context.default());
+  });
+
+  test('load resolves when document is created from binary', async ({ expect }) => {
+    const level = await createLevel();
+    const host = await setupAutomergeHost({ level });
+
+    const document = Automerge.from({ text: 'Hello world' });
+    const binary = Automerge.save(document);
+    const { documentId } = parseAutomergeUrl(generateAutomergeUrl());
+
+    const loadPromise = host.loadDoc(Context.default(), documentId);
+
+    const createdHandle = await host.createDoc(binary, { preserveHistory: true, documentId });
+
+    const loadedHandle = await loadPromise;
+    expect(loadedHandle.doc()).toEqual(createdHandle.doc());
+  });
+
+  test('query single document heads', async ({ expect }) => {
+    const tmpPath = `/tmp/dxos-${PublicKey.random().toHex()}`;
+
+    const level = await createLevel(tmpPath);
+    const host = await setupAutomergeHost({ level });
+    const handle = await host.createDoc({ text: 'Hello world' });
+    const expectedHeads = getHeads(handle.doc()!);
+    await host.flush(Context.default());
+
+    expect(await host.getHeads([handle.documentId])).toEqual([expectedHeads]);
+    await host.close();
+    await level.close();
+
+    {
+      const host = await setupAutomergeHost({ level: await createLevel(tmpPath) });
+      expect(await host.getHeads([handle.documentId])).toEqual([expectedHeads]);
+    }
+  });
+
+  test('query multiple document heads', async ({ expect }) => {
+    const tmpPath = `/tmp/dxos-${PublicKey.random().toHex()}`;
+
+    const level = await createLevel(tmpPath);
+    const host = await setupAutomergeHost({ level });
+    const handles = await Promise.all(range(2, () => host.createDoc({ text: 'Hello world' })));
+    const expectedHeads: (Heads | undefined)[] = handles.map((handle) => getHeads(handle.doc()!));
+    await host.flush(Context.default());
+
+    const ids = handles.map((handle) => handle.documentId);
+    ids.splice(1, 0, 'non-existent-id' as DocumentId);
+    expectedHeads.splice(1, 0, undefined);
+
+    expect(await host.getHeads(ids)).toEqual(expectedHeads);
+    await host.close();
+    await level.close();
+
+    {
+      const level = await createLevel(tmpPath);
+      const host = await setupAutomergeHost({ level });
+      expect(await host.getHeads(ids)).toEqual(expectedHeads);
+      await host.close();
+      await level.close();
+    }
+  });
+
+  test('loads remote document over replication network', { timeout: 5_000 }, async ({ expect }) => {
+    const level1 = await createLevel();
+    const host1 = await setupAutomergeHost({ level: level1 });
+
+    const level2 = await createLevel();
+    const host2 = await setupAutomergeHost({ level: level2 });
+    const handle = await host2.createDoc({ text: 'Hello from Subduction' });
+    await host2.flush(Context.default());
+    await waitForSubductionSave();
+
+    const network = await new TestReplicationNetwork().open();
+    try {
+      await host1.addReplicator(Context.default(), await network.createReplicator());
+      await host2.addReplicator(Context.default(), await network.createReplicator());
+
+      const loaded = await host1.loadDoc<{ text: string }>(Context.default(), handle.documentId, { timeout: 1_000 });
+      expect(loaded.doc()!.text).toEqual('Hello from Subduction');
+    } finally {
+      await host1.close();
+      await host2.close();
+      await network.close();
+    }
+  });
+
+  test('collection synchronization', { timeout: 5_000 }, async ({ expect }) => {
+    const NUM_DOCUMENTS = 10;
+
+    const level1 = await createLevel();
+    const host1 = await setupAutomergeHost({ level: level1 });
+
+    const level2 = await createLevel();
+    const host2 = await setupAutomergeHost({ level: level2 });
+    const documentIds: DocumentId[] = [];
+    for (const i of range(NUM_DOCUMENTS)) {
+      const handle = await host2.createDoc({ docIndex: i });
+      documentIds.push(handle.documentId);
+    }
+    await host2.flush(Context.default());
+    await waitForSubductionSave();
+
+    const network = await new TestReplicationNetwork().open();
+    try {
+      await host1.addReplicator(Context.default(), await network.createReplicator());
+      await host2.addReplicator(Context.default(), await network.createReplicator());
+
+      const collectionId = 'test-collection';
+      await host1.updateLocalCollectionState(collectionId, documentIds);
+      await host2.updateLocalCollectionState(collectionId, documentIds);
+
+      for (const documentId of documentIds) {
+        await expect
+          .poll(() => host1.getHeads([documentId]), { timeout: 1_000 })
+          .toEqual(await host2.getHeads([documentId]));
+      }
+    } finally {
+      await host1.close();
+      await host2.close();
+      await network.close();
+    }
+  });
+
+  test('collection synchronization is bidirectional', { timeout: 10_000 }, async ({ expect }) => {
+    const host1DocumentIds: DocumentId[] = [];
+    const host2DocumentIds: DocumentId[] = [];
+
+    const level1 = await createLevel();
+    const host1 = await setupAutomergeHost({ level: level1 });
+    for (const i of range(3)) {
+      const handle = await host1.createDoc({ host: 1, docIndex: i });
+      host1DocumentIds.push(handle.documentId);
+    }
+    await host1.flush(Context.default());
+
+    const level2 = await createLevel();
+    const host2 = await setupAutomergeHost({ level: level2 });
+    for (const i of range(3)) {
+      const handle = await host2.createDoc({ host: 2, docIndex: i });
+      host2DocumentIds.push(handle.documentId);
+    }
+    await host2.flush(Context.default());
+    await waitForSubductionSave();
+
+    const network = await new TestReplicationNetwork().open();
+    try {
+      await host1.addReplicator(Context.default(), await network.createReplicator());
+      await host2.addReplicator(Context.default(), await network.createReplicator());
+
+      const host1CollectionId = 'test-host1-collection';
+      await host1.updateLocalCollectionState(host1CollectionId, host1DocumentIds);
+      await host2.updateLocalCollectionState(host1CollectionId, host1DocumentIds);
+
+      for (const documentId of host1DocumentIds) {
+        await expect
+          .poll(() => host2.getHeads([documentId]), { timeout: 1_000 })
+          .toEqual(await host1.getHeads([documentId]));
+      }
+
+      const host2CollectionId = 'test-host2-collection';
+      await host1.updateLocalCollectionState(host2CollectionId, host2DocumentIds);
+      await host2.updateLocalCollectionState(host2CollectionId, host2DocumentIds);
+
+      for (const documentId of host2DocumentIds) {
+        await expect
+          .poll(() => host1.getHeads([documentId]), { timeout: 1_000 })
+          .toEqual(await host2.getHeads([documentId]));
+      }
+    } finally {
+      await host1.close();
+      await host2.close();
+      await network.close();
+    }
+  });
+});
+
+const createLevel = async (tmpPath?: string) => {
+  const level = createTestLevel(tmpPath);
+  await openAndClose(level);
+  return level;
+};
+
+const waitForSubductionSave = async () => {
+  await sleep(150);
+};
+
+const setupAutomergeHost = async ({ level }: { level: LevelDB }) => {
+  const host = new AutomergeHost({
+    db: level,
+    useSubduction: true,
+  });
+  await host.open();
+  onTestFinished(async () => {
+    await host.close();
+  });
+  return host;
+};

package/src/automerge/automerge-host.test.ts

@@ -2,127 +2,154 @@
 // Copyright 2023 DXOS.org
 //
 
-import { getHeads } from '@automerge/automerge';
-import type { DocumentId, Heads } from '@automerge/automerge-repo';
-import { onTestFinished, describe, expect, test } from 'vitest';
+import * as A from '@automerge/automerge';
+import { type DocumentId, type Heads, generateAutomergeUrl, parseAutomergeUrl } from '@automerge/automerge-repo';
+import { describe, expect, onTestFinished, test } from 'vitest';
 
-import { IndexMetadataStore } from '@dxos/indexing';
+import { Context } from '@dxos/context';
+import { PublicKey } from '@dxos/keys';
 import type { LevelDB } from '@dxos/kv-store';
 import { createTestLevel } from '@dxos/kv-store/testing';
 import { openAndClose } from '@dxos/test-utils';
 import { range } from '@dxos/util';
 
-import { AutomergeHost } from './automerge-host';
 import { TestReplicationNetwork } from '../testing';
+import { AutomergeHost } from './automerge-host';
 
 describe('AutomergeHost', () => {
   test('can create documents', async () => {
     const level = await createLevel();
     const host = await setupAutomergeHost({ level });
-    const handle = host.repo.create<any>();
+    const handle = await host.createDoc<any>();
     handle.change((doc: any) => {
       doc.text = 'Hello world';
     });
-    await host.repo.flush();
+    await host.flush(Context.default());
     expect(handle.doc()!.text).toEqual('Hello world');
   });
 
   test('changes are preserved in storage', async () => {
     const level = await createLevel();
-
     const host = await setupAutomergeHost({ level });
-    const handle = host.repo.create();
+    const handle = await host.createDoc<any>();
     handle.change((doc: any) => {
       doc.text = 'Hello world';
     });
     const url = handle.url;
 
-    await host.repo.flush();
+    await host.flush(Context.default());
     await host.close();
 
     const host2 = await setupAutomergeHost({ level });
-    const handle2 = await host2.repo.find<any>(url);
+    const handle2 = await host2.loadDoc<any>(Context.default(), url);
     await handle2.whenReady();
     expect(handle2.doc()!.text).toEqual('Hello world');
-    await host2.repo.flush();
+    await host2.flush(Context.default());
   });
 
-  test('query single document heads', async () => {
+  test('load resolves when document is created from binary', async () => {
     const level = await createLevel();
+    const host = await setupAutomergeHost({ level });
+
+    // Create a document to get its binary representation
+    const document = A.from({ text: 'Hello world' });
+    const binary = A.save(document);
+    const { documentId } = parseAutomergeUrl(generateAutomergeUrl());
+
+    // Start loading a non-existent document (should hang until created)
+    const loadPromise = host.loadDoc(Context.default(), documentId);
+
+    // Create the document from binary - this should resolve the load
+    const createdHandle = await host.createDoc(binary, { preserveHistory: true, documentId });
+
+    // The load should now resolve
+    const loadedHandle = await loadPromise;
+    expect(loadedHandle.doc()).toEqual(createdHandle.doc());
+  });
 
+  test('query single document heads', async () => {
+    const tmpPath = `/tmp/dxos-${PublicKey.random().toHex()}`;
+
+    const level = await createLevel(tmpPath);
     const host = await setupAutomergeHost({ level });
-    const handle = host.createDoc({ text: 'Hello world' });
-    const expectedHeads = getHeads(handle.doc()!);
-    await host.flush();
+    const handle = await host.createDoc({ text: 'Hello world' });
+    const expectedHeads = A.getHeads(handle.doc()!);
+    await host.flush(Context.default());
 
     expect(await host.getHeads([handle.documentId])).toEqual([expectedHeads]);
+    await host.close();
+    await level.close();
 
     // Simulate a restart.
     {
-      const host = await setupAutomergeHost({ level });
+      const host = await setupAutomergeHost({ level: await createLevel(tmpPath) });
       expect(await host.getHeads([handle.documentId])).toEqual([expectedHeads]);
     }
   });
 
   test('query multiple document heads', async () => {
-    const level = await createLevel();
+    const tmpPath = `/tmp/dxos-${PublicKey.random().toHex()}`;
 
+    const level = await createLevel(tmpPath);
     const host = await setupAutomergeHost({ level });
-    const handles = range(2, () => host.createDoc({ text: 'Hello world' }));
-    const expectedHeads: (Heads | undefined)[] = handles.map((handle) => getHeads(handle.doc()!));
-    await host.flush();
+    const handles = await Promise.all(range(2, () => host.createDoc({ text: 'Hello world' })));
+    const expectedHeads: (Heads | undefined)[] = handles.map((handle) => A.getHeads(handle.doc()!));
+    await host.flush(Context.default());
 
     const ids = handles.map((handle) => handle.documentId);
     ids.splice(1, 0, 'non-existent-id' as DocumentId);
     expectedHeads.splice(1, 0, undefined);
 
     expect(await host.getHeads(ids)).toEqual(expectedHeads);
+    await host.close();
+    await level.close();
 
     // Simulate a restart.
     {
+      const level = await createLevel(tmpPath);
       const host = await setupAutomergeHost({ level });
       expect(await host.getHeads(ids)).toEqual(expectedHeads);
+      await host.close();
+      await level.close();
     }
   });
 
-  test('collection synchronization', async () => {
+  test('collection synchronization', { timeout: 30_000 }, async () => {
     const NUM_DOCUMENTS = 10;
 
     const level1 = await createLevel();
     const host1 = await setupAutomergeHost({ level: level1 });
 
     const level2 = await createLevel();
+    const host2 = await setupAutomergeHost({ level: level2 });
     const documentIds: DocumentId[] = [];
-    {
-      const host2 = await setupAutomergeHost({ level: level2 });
-      for (const i of range(NUM_DOCUMENTS)) {
-        const handle = host2.createDoc({ docIndex: i });
-        documentIds.push(handle.documentId);
-      }
-      await host2.flush();
-      await host2.close();
+    for (const i of range(NUM_DOCUMENTS)) {
+      const handle = await host2.createDoc({ docIndex: i });
+      documentIds.push(handle.documentId);
     }
+    await host2.flush(Context.default());
 
-    const host2 = await setupAutomergeHost({ level: level2 });
+    const network = await new TestReplicationNetwork().open();
+    await host1.addReplicator(Context.default(), await network.createReplicator());
+    await host2.addReplicator(Context.default(), await network.createReplicator());
 
     const collectionId = 'test-collection';
     await host1.updateLocalCollectionState(collectionId, documentIds);
     await host2.updateLocalCollectionState(collectionId, documentIds);
 
-    await using network = await new TestReplicationNetwork().open();
-    await host1.addReplicator(await network.createReplicator());
-    await host2.addReplicator(await network.createReplicator());
-
     for (const documentId of documentIds) {
-      await expect.poll(() => host1.getHeads([documentId])).toEqual(await host2.getHeads([documentId]));
+      await expect
+        .poll(() => host1.getHeads([documentId]), { timeout: 20_000 })
+        .toEqual(await host2.getHeads([documentId]));
     }
 
     await host1.close();
     await host2.close();
+    await network.close();
   });
 
-  const createLevel = async () => {
-    const level = createTestLevel();
+  const createLevel = async (tmpPath?: string) => {
+    const level = createTestLevel(tmpPath);
     await openAndClose(level);
     return level;
   };
@@ -131,7 +158,6 @@ describe('AutomergeHost', () => {
 const setupAutomergeHost = async ({ level }: { level: LevelDB }) => {
   const host = new AutomergeHost({
     db: level,
-    indexMetadataStore: new IndexMetadataStore({ db: level.sublevel('index-metadata') }),
   });
   await host.open();
   onTestFinished(async () => {