@dxos/echo-pipeline 0.8.3 → 0.8.4-main.05e74ebcff

package/package.json

@@ -1,38 +1,35 @@
 {
   "name": "@dxos/echo-pipeline",
-  "version": "0.8.3",
+  "version": "0.8.4-main.05e74ebcff",
   "description": "ECHO database.",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
-  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dxos/dxos"
+  },
+  "license": "FSL-1.1-Apache-2.0",
   "author": "info@dxos.org",
-  "sideEffects": true,
+  "sideEffects": false,
   "type": "module",
   "exports": {
     ".": {
+      "source": "./src/index.ts",
       "types": "./dist/types/src/index.d.ts",
-      "browser": "./dist/lib/browser/index.mjs",
-      "node": "./dist/lib/node-esm/index.mjs"
+      "default": "./dist/lib/neutral/index.mjs"
     },
     "./testing": {
+      "source": "./src/testing/index.ts",
       "types": "./dist/types/src/testing/index.d.ts",
-      "browser": "./dist/lib/browser/testing/index.mjs",
-      "node": "./dist/lib/node-esm/testing/index.mjs"
+      "default": "./dist/lib/neutral/testing/index.mjs"
     },
     "./filter": {
+      "source": "./src/filter/index.ts",
       "types": "./dist/types/src/filter/index.d.ts",
-      "browser": "./dist/lib/browser/filter/index.mjs",
-      "node": "./dist/lib/node-esm/filter/index.mjs"
+      "default": "./dist/lib/neutral/filter/index.mjs"
     }
   },
   "types": "dist/types/src/index.d.ts",
-  "typesVersions": {
-    "*": {
-      "testing": [
-        "dist/types/src/testing/index.d.ts"
-      ]
-    }
-  },
   "files": [
     "testing.d.ts",
     "testing.js",
@@ -40,51 +37,58 @@
     "src"
   ],
   "dependencies": {
-    "@automerge/automerge": "3.0.0-beta.4",
-    "@automerge/automerge-repo": "2.0.1",
+    "@automerge/automerge": "3.2.6",
+    "@automerge/automerge-repo": "2.6.0-subduction.17",
+    "@automerge/automerge-subduction": "0.12.1",
+    "@effect/sql": "0.49.0",
+    "@effect/sql-sqlite-node": "0.50.1",
+    "bs58check": "^4.0.0",
     "crc-32": "^1.2.2",
-    "effect": "3.14.21",
+    "effect": "3.20.0",
+    "fast-deep-equal": "^3.1.3",
     "level-transcoder": "^1.0.1",
-    "lodash.isequal": "^4.5.0",
-    "@dxos/codec-protobuf": "0.8.3",
-    "@dxos/credentials": "0.8.3",
-    "@dxos/context": "0.8.3",
-    "@dxos/async": "0.8.3",
-    "@dxos/crypto": "0.8.3",
-    "@dxos/debug": "0.8.3",
-    "@dxos/echo": "0.8.3",
-    "@dxos/echo-protocol": "0.8.3",
-    "@dxos/edge-client": "0.8.3",
-    "@dxos/echo-schema": "0.8.3",
-    "@dxos/feed-store": "0.8.3",
-    "@dxos/errors": "0.8.3",
-    "@dxos/indexing": "0.8.3",
-    "@dxos/hypercore": "0.8.3",
-    "@dxos/invariant": "0.8.3",
-    "@dxos/keyring": "0.8.3",
-    "@dxos/keys": "0.8.3",
-    "@dxos/kv-store": "0.8.3",
-    "@dxos/log": "0.8.3",
-    "@dxos/messaging": "0.8.3",
-    "@dxos/network-manager": "0.8.3",
-    "@dxos/node-std": "0.8.3",
-    "@dxos/teleport": "0.8.3",
-    "@dxos/protocols": "0.8.3",
-    "@dxos/random-access-storage": "0.8.3",
-    "@dxos/teleport-extension-automerge-replicator": "0.8.3",
-    "@dxos/teleport-extension-gossip": "0.8.3",
-    "@dxos/teleport-extension-object-sync": "0.8.3",
-    "@dxos/teleport-extension-replicator": "0.8.3",
-    "@dxos/timeframe": "0.8.3",
-    "@dxos/tracing": "0.8.3",
-    "@dxos/util": "0.8.3",
-    "@dxos/typings": "0.8.3"
+    "semver": "^7.7.3",
+    "@dxos/async": "0.8.4-main.05e74ebcff",
+    "@dxos/codec-protobuf": "0.8.4-main.05e74ebcff",
+    "@dxos/context": "0.8.4-main.05e74ebcff",
+    "@dxos/credentials": "0.8.4-main.05e74ebcff",
+    "@dxos/debug": "0.8.4-main.05e74ebcff",
+    "@dxos/echo": "0.8.4-main.05e74ebcff",
+    "@dxos/echo-protocol": "0.8.4-main.05e74ebcff",
+    "@dxos/edge-client": "0.8.4-main.05e74ebcff",
+    "@dxos/crypto": "0.8.4-main.05e74ebcff",
+    "@dxos/errors": "0.8.4-main.05e74ebcff",
+    "@dxos/effect": "0.8.4-main.05e74ebcff",
+    "@dxos/feed-store": "0.8.4-main.05e74ebcff",
+    "@dxos/feed": "0.8.4-main.05e74ebcff",
+    "@dxos/hypercore": "0.8.4-main.05e74ebcff",
+    "@dxos/index-core": "0.8.4-main.05e74ebcff",
+    "@dxos/invariant": "0.8.4-main.05e74ebcff",
+    "@dxos/keys": "0.8.4-main.05e74ebcff",
+    "@dxos/keyring": "0.8.4-main.05e74ebcff",
+    "@dxos/kv-store": "0.8.4-main.05e74ebcff",
+    "@dxos/log": "0.8.4-main.05e74ebcff",
+    "@dxos/messaging": "0.8.4-main.05e74ebcff",
+    "@dxos/network-manager": "0.8.4-main.05e74ebcff",
+    "@dxos/node-std": "0.8.4-main.05e74ebcff",
+    "@dxos/protocols": "0.8.4-main.05e74ebcff",
+    "@dxos/sql-sqlite": "0.8.4-main.05e74ebcff",
+    "@dxos/teleport": "0.8.4-main.05e74ebcff",
+    "@dxos/random-access-storage": "0.8.4-main.05e74ebcff",
+    "@dxos/teleport-extension-automerge-replicator": "0.8.4-main.05e74ebcff",
+    "@dxos/teleport-extension-gossip": "0.8.4-main.05e74ebcff",
+    "@dxos/teleport-extension-object-sync": "0.8.4-main.05e74ebcff",
+    "@dxos/timeframe": "0.8.4-main.05e74ebcff",
+    "@dxos/typings": "0.8.4-main.05e74ebcff",
+    "@dxos/tracing": "0.8.4-main.05e74ebcff",
+    "@dxos/util": "0.8.4-main.05e74ebcff",
+    "@dxos/teleport-extension-replicator": "0.8.4-main.05e74ebcff"
   },
   "devDependencies": {
-    "@types/lodash.isequal": "^4.5.0",
+    "@types/semver": "^7.7.1",
     "fast-check": "^3.19.0",
     "get-port-please": "^3.1.1",
-    "@dxos/test-utils": "0.8.3"
+    "@dxos/test-utils": "0.8.4-main.05e74ebcff"
   },
   "publishConfig": {
     "access": "public"

package/src/automerge/automerge-host-subduction.test.ts

@@ -0,0 +1,263 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { getHeads } from '@automerge/automerge';
+import * as Automerge from '@automerge/automerge';
+import { type DocumentId, type Heads, generateAutomergeUrl, parseAutomergeUrl } from '@automerge/automerge-repo';
+import { describe, onTestFinished, test } from 'vitest';
+
+import { sleep } from '@dxos/async';
+import { Context } from '@dxos/context';
+import type { CollectionId } from '@dxos/echo-protocol';
+import { invariant } from '@dxos/invariant';
+import { PublicKey } from '@dxos/keys';
+import type { LevelDB } from '@dxos/kv-store';
+import { createTestLevel } from '@dxos/kv-store/testing';
+import { openAndClose } from '@dxos/test-utils';
+import { range } from '@dxos/util';
+
+import { TestReplicationNetwork } from '../testing';
+import { AutomergeHost } from './automerge-host';
+
+describe('AutomergeHost with Subduction', () => {
+  test('can create documents', async ({ expect }) => {
+    const level = await createLevel();
+    const host = await setupAutomergeHost({ level });
+    const handle = await host.createDoc<any>();
+    handle.change((doc: any) => {
+      doc.text = 'Hello world';
+    });
+    await host.flush(Context.default());
+    expect(handle.doc()!.text).toEqual('Hello world');
+  });
+
+  test('changes are preserved in storage', async ({ expect }) => {
+    const level = await createLevel();
+    const host = await setupAutomergeHost({ level });
+    const handle = await host.createDoc<any>();
+    handle.change((doc: any) => {
+      doc.text = 'Hello world';
+    });
+    const url = handle.url;
+
+    await host.flush(Context.default());
+    await host.close();
+
+    const host2 = await setupAutomergeHost({ level });
+    const handle2 = await host2.loadDoc<any>(Context.default(), url);
+    invariant(handle2);
+    await handle2.whenReady();
+    expect(handle2.doc()!.text).toEqual('Hello world');
+    await host2.flush(Context.default());
+  });
+
+  test('load resolves when document is created from binary', async ({ expect }) => {
+    const level = await createLevel();
+    const host = await setupAutomergeHost({ level });
+
+    const document = Automerge.from({ text: 'Hello world' });
+    const binary = Automerge.save(document);
+    const { documentId } = parseAutomergeUrl(generateAutomergeUrl());
+
+    const loadPromise = host.loadDoc(Context.default(), documentId);
+
+    const createdHandle = await host.createDoc(binary, { preserveHistory: true, documentId });
+
+    const loadedHandle = await loadPromise;
+    invariant(loadedHandle);
+    expect(loadedHandle.doc()).toEqual(createdHandle.doc());
+  });
+
+  test('query single document heads', async ({ expect }) => {
+    const tmpPath = `/tmp/dxos-${PublicKey.random().toHex()}`;
+
+    const level = await createLevel(tmpPath);
+    const host = await setupAutomergeHost({ level });
+    const handle = await host.createDoc({ text: 'Hello world' });
+    const expectedHeads = getHeads(handle.doc()!);
+    await host.flush(Context.default());
+
+    expect(await host.getHeads([handle.documentId])).toEqual([expectedHeads]);
+    await host.close();
+    await level.close();
+
+    {
+      const host = await setupAutomergeHost({ level: await createLevel(tmpPath) });
+      expect(await host.getHeads([handle.documentId])).toEqual([expectedHeads]);
+    }
+  });
+
+  test('query multiple document heads', async ({ expect }) => {
+    const tmpPath = `/tmp/dxos-${PublicKey.random().toHex()}`;
+
+    const level = await createLevel(tmpPath);
+    const host = await setupAutomergeHost({ level });
+    const handles = await Promise.all(range(2, () => host.createDoc({ text: 'Hello world' })));
+    const expectedHeads: (Heads | undefined)[] = handles.map((handle) => getHeads(handle.doc()!));
+    await host.flush(Context.default());
+
+    const ids = handles.map((handle) => handle.documentId);
+    ids.splice(1, 0, 'non-existent-id' as DocumentId);
+    expectedHeads.splice(1, 0, undefined);
+
+    expect(await host.getHeads(ids)).toEqual(expectedHeads);
+    await host.close();
+    await level.close();
+
+    {
+      const level = await createLevel(tmpPath);
+      const host = await setupAutomergeHost({ level });
+      expect(await host.getHeads(ids)).toEqual(expectedHeads);
+      await host.close();
+      await level.close();
+    }
+  });
+
+  test('loads remote document over replication network', { timeout: 5_000 }, async ({ expect }) => {
+    const level1 = await createLevel();
+    const host1 = await setupAutomergeHost({ level: level1 });
+
+    const level2 = await createLevel();
+    const host2 = await setupAutomergeHost({ level: level2 });
+    const handle = await host2.createDoc({ text: 'Hello from Subduction' });
+    await host2.flush(Context.default());
+    await waitForSubductionSave();
+
+    const network = await new TestReplicationNetwork().open();
+    try {
+      await host1.addReplicator(Context.default(), await network.createReplicator());
+      await host2.addReplicator(Context.default(), await network.createReplicator());
+
+      const loaded = await host1.loadDoc<{ text: string }>(Context.default(), handle.documentId, { timeout: 1_000 });
+      invariant(loaded);
+      expect(loaded.doc()!.text).toEqual('Hello from Subduction');
+    } finally {
+      await host1.close();
+      await host2.close();
+      await network.close();
+    }
+  });
+
+  test('collection synchronization', { timeout: 5_000 }, async ({ expect }) => {
+    const NUM_DOCUMENTS = 10;
+
+    const level1 = await createLevel();
+    const host1 = await setupAutomergeHost({ level: level1 });
+
+    const level2 = await createLevel();
+    const host2 = await setupAutomergeHost({ level: level2 });
+    const documentIds: DocumentId[] = [];
+    for (const i of range(NUM_DOCUMENTS)) {
+      const handle = await host2.createDoc({ docIndex: i });
+      documentIds.push(handle.documentId);
+    }
+    await host2.flush(Context.default());
+    await waitForSubductionSave();
+
+    const network = await new TestReplicationNetwork().open();
+    try {
+      await host1.addReplicator(Context.default(), await network.createReplicator());
+      await host2.addReplicator(Context.default(), await network.createReplicator());
+
+      const collectionId = 'test-collection';
+      const collectionUpdates: CollectionId[] = [];
+      const unsubscribe = host1.collectionStateUpdated.on(({ collectionId }) => {
+        collectionUpdates.push(collectionId);
+      });
+      await host1.updateLocalCollectionState(collectionId, documentIds);
+      await host2.updateLocalCollectionState(collectionId, documentIds);
+
+      for (const documentId of documentIds) {
+        await expect
+          .poll(() => host1.getHeads([documentId]), { timeout: 1_000 })
+          .toEqual(await host2.getHeads([documentId]));
+      }
+
+      // collectionStateUpdated fires whenever host1 observes a `collection-state` from host2
+      // (see {@link CollectionSynchronizer.remoteStateUpdated}). At least one event must
+      // arrive for the heads to have converged via the collection-sync path rather than
+      // raw Subduction byte transport.
+      expect(collectionUpdates).toContain(collectionId as CollectionId);
+      unsubscribe();
+    } finally {
+      await host1.close();
+      await host2.close();
+      await network.close();
+    }
+  });
+
+  test('collection synchronization is bidirectional', { timeout: 10_000 }, async ({ expect }) => {
+    const host1DocumentIds: DocumentId[] = [];
+    const host2DocumentIds: DocumentId[] = [];
+
+    const level1 = await createLevel();
+    const host1 = await setupAutomergeHost({ level: level1 });
+    for (const i of range(3)) {
+      const handle = await host1.createDoc({ host: 1, docIndex: i });
+      host1DocumentIds.push(handle.documentId);
+    }
+    await host1.flush(Context.default());
+
+    const level2 = await createLevel();
+    const host2 = await setupAutomergeHost({ level: level2 });
+    for (const i of range(3)) {
+      const handle = await host2.createDoc({ host: 2, docIndex: i });
+      host2DocumentIds.push(handle.documentId);
+    }
+    await host2.flush(Context.default());
+    await waitForSubductionSave();
+
+    const network = await new TestReplicationNetwork().open();
+    try {
+      await host1.addReplicator(Context.default(), await network.createReplicator());
+      await host2.addReplicator(Context.default(), await network.createReplicator());
+
+      const host1CollectionId = 'test-host1-collection';
+      await host1.updateLocalCollectionState(host1CollectionId, host1DocumentIds);
+      await host2.updateLocalCollectionState(host1CollectionId, host1DocumentIds);
+
+      for (const documentId of host1DocumentIds) {
+        await expect
+          .poll(() => host2.getHeads([documentId]), { timeout: 1_000 })
+          .toEqual(await host1.getHeads([documentId]));
+      }
+
+      const host2CollectionId = 'test-host2-collection';
+      await host1.updateLocalCollectionState(host2CollectionId, host2DocumentIds);
+      await host2.updateLocalCollectionState(host2CollectionId, host2DocumentIds);
+
+      for (const documentId of host2DocumentIds) {
+        await expect
+          .poll(() => host1.getHeads([documentId]), { timeout: 1_000 })
+          .toEqual(await host2.getHeads([documentId]));
+      }
+    } finally {
+      await host1.close();
+      await host2.close();
+      await network.close();
+    }
+  });
+});
+
+const createLevel = async (tmpPath?: string) => {
+  const level = createTestLevel(tmpPath);
+  await openAndClose(level);
+  return level;
+};
+
+const waitForSubductionSave = async () => {
+  await sleep(150);
+};
+
+const setupAutomergeHost = async ({ level }: { level: LevelDB }) => {
+  const host = new AutomergeHost({
+    db: level,
+    useSubduction: true,
+  });
+  await host.open();
+  onTestFinished(async () => {
+    await host.close();
+  });
+  return host;
+};