@dxos/echo-doc 0.0.0

package/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@dxos/echo-doc",
+  "version": "0.0.0",
+  "description": "Opinionated Automerge document wrapper for ECHO.",
+  "homepage": "https://dxos.org",
+  "bugs": "https://github.com/dxos/dxos/issues",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dxos/dxos"
+  },
+  "license": "FSL-1.1-Apache-2.0",
+  "author": "DXOS.org",
+  "sideEffects": false,
+  "type": "module",
+  "exports": {
+    ".": {
+      "source": "./src/index.ts",
+      "types": "./dist/types/src/index.d.ts",
+      "default": "./dist/lib/neutral/index.mjs"
+    }
+  },
+  "types": "dist/types/src/index.d.ts",
+  "files": [
+    "dist",
+    "src"
+  ],
+  "dependencies": {
+    "@automerge/automerge": "catalog:",
+    "@dxos/echo": "workspace:*",
+    "@dxos/echo-client": "workspace:*",
+    "@dxos/invariant": "workspace:*",
+    "@dxos/log": "workspace:*",
+    "@dxos/util": "workspace:*",
+    "effect": "catalog:"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/src/Doc.test.ts

@@ -0,0 +1,98 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { next as A } from '@automerge/automerge';
+import { describe, test } from 'vitest';
+
+import { Obj } from '@dxos/echo';
+import { EchoTestBuilder } from '@dxos/echo-client/testing';
+import { TestSchema } from '@dxos/echo/testing';
+
+import * as Doc from './Doc';
+import { applyEdits } from './edits';
+
+// `Doc.createAccessor` resolves an accessor for either backend: a database-attached object (its core
+// is the space document) or an in-memory `Obj.make` object (a local core is materialized on demand).
+
+describe('Doc', () => {
+  describe('echo-db backend', () => {
+    test('reads and edits an attached object', async ({ expect }) => {
+      const builder = new EchoTestBuilder();
+      const { db, graph } = await builder.createDatabase();
+      graph.registry.add([TestSchema.Task]);
+
+      const obj = db.add(Obj.make(TestSchema.Task, { description: 'hello' }));
+      const accessor = Doc.createAccessor(obj, ['description']);
+      expect(Doc.getValue<string>(accessor)).toBe('hello');
+
+      accessor.handle.change((doc) => {
+        const path: A.Prop[] = [...accessor.path];
+        A.splice(doc, path, 5, 0, ' world');
+      });
+      expect(obj.description).toBe('hello world');
+    });
+  });
+
+  describe('in-memory backend', () => {
+    test('binds before db.add and edits survive attach', async ({ expect }) => {
+      const obj = Obj.make(TestSchema.Task, { description: 'hello' });
+
+      // No db: a local core is materialized so the accessor can bind.
+      const accessor = Doc.createAccessor(obj, ['description']);
+      expect(Doc.getValue<string>(accessor)).toBe('hello');
+
+      accessor.handle.change((doc) => {
+        const path: A.Prop[] = [...accessor.path];
+        A.splice(doc, path, 5, 0, ' world');
+      });
+      expect(Doc.getValue<string>(accessor)).toBe('hello world');
+
+      const builder = new EchoTestBuilder();
+      const { db, graph } = await builder.createDatabase();
+      graph.registry.add([TestSchema.Task]);
+      const added = db.add(obj);
+      expect(added).toBe(obj);
+      expect(added.description).toBe('hello world');
+    });
+  });
+
+  describe('applyEdits', () => {
+    test('applies edits over an attached object', async ({ expect }) => {
+      const builder = new EchoTestBuilder();
+      const { db, graph } = await builder.createDatabase();
+      graph.registry.add([TestSchema.Task]);
+
+      const obj = db.add(Obj.make(TestSchema.Task, { description: 'the cat sat' }));
+      const accessor = Doc.createAccessor(obj, ['description']);
+      expect(applyEdits(accessor, [{ oldString: 'cat', newString: 'dog' }])).toBe('the dog sat');
+      expect(obj.description).toBe('the dog sat');
+    });
+
+    test('applies edits over an in-memory object', ({ expect }) => {
+      const obj = Obj.make(TestSchema.Task, { description: 'the cat sat' });
+      const accessor = Doc.createAccessor(obj, ['description']);
+      expect(applyEdits(accessor, [{ oldString: 'cat', newString: 'dog' }])).toBe('the dog sat');
+    });
+
+    test('replaces all occurrences with replaceAll', ({ expect }) => {
+      const obj = Obj.make(TestSchema.Task, { description: 'the cat sat on the cat' });
+      const accessor = Doc.createAccessor(obj, ['description']);
+      expect(applyEdits(accessor, [{ oldString: 'cat', newString: 'dog', replaceAll: true }])).toBe(
+        'the dog sat on the dog',
+      );
+    });
+
+    test('throws when oldString is not found', ({ expect }) => {
+      const obj = Obj.make(TestSchema.Task, { description: 'hello' });
+      const accessor = Doc.createAccessor(obj, ['description']);
+      expect(() => applyEdits(accessor, [{ oldString: 'xyz', newString: 'abc' }])).toThrow();
+    });
+
+    test('throws on an empty oldString (would otherwise loop forever with replaceAll)', ({ expect }) => {
+      const obj = Obj.make(TestSchema.Task, { description: 'hello' });
+      const accessor = Doc.createAccessor(obj, ['description']);
+      expect(() => applyEdits(accessor, [{ oldString: '', newString: 'x', replaceAll: true }])).toThrow();
+    });
+  });
+});

package/src/Doc.ts

@@ -0,0 +1,58 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+// @import-as-namespace
+
+import { next as A } from '@automerge/automerge';
+
+import { Doc, createObject, getObjectCore, isEchoObject } from '@dxos/echo-client';
+import { type AnyProperties, isProxy } from '@dxos/echo/internal';
+import { assertArgument } from '@dxos/invariant';
+
+// TODO(burdon): Reduce the dependency on @dxos/echo-client (e.g. move ObjectCore lower or invert).
+
+export type KeyPath = Doc.KeyPath;
+
+export const isKeyPath = Doc.isKeyPath;
+
+export type Handle<T = any> = Doc.Handle<T>;
+
+export type Accessor<T = any> = Doc.Accessor<T>;
+
+export const getValue = Doc.getValue;
+
+/**
+ * Resolves an {@link Accessor} for a value within an object, agnostic to whether the object is
+ * in-memory or attached to a database. An `Obj.make` object has no `ObjectCore` until `db.add`,
+ * so an in-memory core is materialized (via `createObject`) to let the accessor bind before
+ * persistence; the object's edits then survive a later `db.add`.
+ */
+export const createAccessor = <T extends AnyProperties>(
+  obj: T,
+  path: KeyPath | Extract<keyof T, string | number>,
+): Accessor<T> => {
+  const keyPath: KeyPath = Array.isArray(path) ? path : [path];
+  assertArgument(isProxy(obj), 'obj', 'expect obj to be a LiveObject');
+  assertArgument(isKeyPath(keyPath), 'path', 'expect path to be a valid key path');
+
+  const live: AnyProperties = isEchoObject(obj) ? obj : createObject(obj);
+  return getObjectCore(live).getDocAccessor(keyPath);
+};
+
+/**
+ * Sets the text value at the given path, applying the change as a minimal Automerge delta (via
+ * `A.updateText`) rather than replacing the whole string. This preserves cursors/anchors and merges
+ * cleanly with concurrent edits — use it to write an edited string back into a collaborative text field.
+ */
+export const updateText = <T extends AnyProperties>(
+  obj: T,
+  path: KeyPath | Extract<keyof T, string | number>,
+  newText: string,
+): T => {
+  const accessor = createAccessor(obj, path);
+  accessor.handle.change((doc) => {
+    A.updateText(doc, accessor.path.slice(), newText);
+  });
+  return obj;
+};

package/src/edits.ts

@@ -0,0 +1,60 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { next as A } from '@automerge/automerge';
+import * as Schema from 'effect/Schema';
+
+import * as Doc from './Doc';
+
+/**
+ * A single find/replace edit applied to a text document. This is the structured form of the diff
+ * protocol described in document skills' instructions.
+ */
+export const Edit = Schema.Struct({
+  oldString: Schema.String.annotations({
+    description: 'The exact text to find and replace.',
+  }),
+  newString: Schema.String.annotations({
+    description: 'The text to replace it with.',
+  }),
+  replaceAll: Schema.optional(Schema.Boolean).annotations({
+    description: 'Replace all occurrences of oldString (default: replace the first occurrence).',
+  }),
+});
+
+export interface Edit extends Schema.Schema.Type<typeof Edit> {}
+
+/**
+ * Applies a sequence of find/replace {@link Edit}s to the string value an accessor points at,
+ * returning the resulting content. Throws if a non-`replaceAll` edit's `oldString` is not found.
+ */
+export const applyEdits = (accessor: Doc.Accessor, edits: readonly Edit[]): string => {
+  for (const edit of edits) {
+    // `''.indexOf('')` is 0 (never -1), so a `replaceAll` with an empty match would loop forever.
+    if (edit.oldString.length === 0) {
+      throw new Error('Edit oldString must be non-empty.');
+    }
+
+    accessor.handle.change((doc) => {
+      const text = Doc.getValue<string>(accessor);
+      // Automerge's `splice` types the path as mutable `Prop[]`; our `KeyPath` is readonly and is not mutated here.
+      if (edit.replaceAll) {
+        let idx = text.indexOf(edit.oldString);
+        while (idx !== -1) {
+          A.splice(doc, accessor.path as A.Prop[], idx, edit.oldString.length, edit.newString);
+          const updated = Doc.getValue<string>(accessor);
+          idx = updated.indexOf(edit.oldString, idx + edit.newString.length);
+        }
+      } else {
+        const idx = text.indexOf(edit.oldString);
+        if (idx === -1) {
+          throw new Error(`Edit not found: ${JSON.stringify(edit.oldString)}`);
+        }
+        A.splice(doc, accessor.path as A.Prop[], idx, edit.oldString.length, edit.newString);
+      }
+    });
+  }
+
+  return Doc.getValue<string>(accessor);
+};

package/src/index.ts

@@ -0,0 +1,13 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * as Doc from './Doc';
+export { Edit, applyEdits } from './edits';
+export {
+  AbstractStoreAdapter,
+  type BaseElement,
+  type Batch,
+  Modified,
+  type StoreAdapterOptions,
+} from './store-adapter';

package/src/record.ts

@@ -0,0 +1,90 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { next as A } from '@automerge/automerge';
+
+import { isNonNullable } from '@dxos/util';
+
+import * as Doc from './Doc';
+
+// Strings longer than this have collaborative editing disabled for performance reasons.
+const STRING_CRDT_LIMIT = 300_000;
+
+/**
+ * Default codec: encode a model value to an Automerge record.
+ */
+export const encode = (value: any): any => {
+  if (Array.isArray(value)) {
+    return value.map(encode);
+  }
+  if (value instanceof A.RawString) {
+    throw new Error('Encode called on automerge data.');
+  }
+  if (typeof value === 'object' && value !== null) {
+    return Object.fromEntries(
+      Object.entries(value)
+        .map(([key, value]) => {
+          const encoded = encode(value);
+          if (encoded === undefined) {
+            return undefined;
+          }
+          return [key, encoded];
+        })
+        .filter(isNonNullable),
+    );
+  }
+  if (typeof value === 'string' && value.length > STRING_CRDT_LIMIT) {
+    return new A.RawString(value);
+  }
+
+  return value;
+};
+
+/**
+ * Default codec: decode an Automerge record to a model value.
+ */
+export const decode = (value: any): any => {
+  if (Array.isArray(value)) {
+    return value.map(decode);
+  }
+  if (value instanceof A.RawString) {
+    return value.toString();
+  }
+  if (typeof value === 'object' && value !== null) {
+    return Object.fromEntries(Object.entries(value).map(([key, value]) => [key, decode(value)]));
+  }
+
+  return value;
+};
+
+/**
+ * Relative path from `base`, or `undefined` if `path` is not under `base`. Used to filter mutations.
+ */
+export const rebasePath = (path: A.Prop[], base: Doc.KeyPath): A.Prop[] | undefined => {
+  if (path.length < base.length) {
+    return undefined;
+  }
+  for (let i = 0; i < base.length; ++i) {
+    if (path[i] !== base[i]) {
+      return undefined;
+    }
+  }
+
+  return path.slice(base.length);
+};
+
+/**
+ * Read the value at `path`. When `init` is set, auto-vivifies intermediate records.
+ */
+export const getDeep = (obj: any, path: Doc.KeyPath, init = false): any => {
+  let value = obj;
+  for (const key of path) {
+    if (init) {
+      value[key] ??= {};
+    }
+    value = value?.[key];
+  }
+
+  return value;
+};

package/src/store-adapter.test.ts

@@ -0,0 +1,100 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+import { describe, test } from 'vitest';
+
+import { DXN, Obj, Type } from '@dxos/echo';
+
+import * as Doc from './Doc';
+import { AbstractStoreAdapter, type Batch } from './store-adapter';
+
+const Canvas = Type.makeObject(DXN.make('com.example.type.canvas', '0.1.0'))(
+  Schema.Struct({
+    content: Schema.optional(Schema.Any),
+  }),
+);
+
+type Element = { id: string; value: string };
+
+// Reads/mutates the element map at the accessor's (namespaced) path.
+const elementMap = (accessor: Doc.Accessor): Record<string, Element> => Doc.getValue<Record<string, Element>>(accessor);
+const mutateMap = (accessor: Doc.Accessor, mutate: (map: Record<string, Element>) => void): void =>
+  accessor.handle.change((doc: any) => {
+    let map = doc;
+    for (const key of accessor.path) {
+      map = map[key];
+    }
+    mutate(map);
+  });
+
+// Minimal adapter that mirrors the document map into a local `Map`.
+class TestAdapter extends AbstractStoreAdapter<Element> {
+  readonly store = new Map<string, Element>();
+  readonly updates: Batch<Element>[] = [];
+
+  override getElements(): Element[] {
+    return [...this.store.values()];
+  }
+
+  protected override onUpdate(batch: Batch<Element>): void {
+    this.updates.push(batch);
+    [...(batch.added ?? []), ...(batch.updated ?? [])].forEach((element) => this.store.set(element.id, element));
+    (batch.deleted ?? []).forEach((id) => this.store.delete(id));
+  }
+
+  write(batch: Batch<Element>): void {
+    this.updateDatabase(batch);
+  }
+}
+
+describe('AbstractStoreAdapter', () => {
+  test('seeds an empty document from the store', ({ expect }) => {
+    const obj = Obj.make(Canvas, { content: {} });
+    const accessor = Doc.createAccessor(obj, ['content']);
+
+    const adapter = new TestAdapter();
+    adapter.store.set('a', { id: 'a', value: '1' });
+    const dispose = adapter.open(accessor);
+
+    expect(elementMap(accessor).a).toEqual({ id: 'a', value: '1' });
+    dispose();
+  });
+
+  test('hydrates the store from a populated document', ({ expect }) => {
+    const obj = Obj.make(Canvas, { content: { a: { id: 'a', value: '1' } } });
+    const accessor = Doc.createAccessor(obj, ['content']);
+
+    const adapter = new TestAdapter();
+    const dispose = adapter.open(accessor);
+
+    expect(adapter.store.get('a')).toEqual({ id: 'a', value: '1' });
+    dispose();
+  });
+
+  test('writes store changes to the document and reflects external changes back', ({ expect }) => {
+    const obj = Obj.make(Canvas, { content: {} });
+    const accessor = Doc.createAccessor(obj, ['content']);
+
+    const adapter = new TestAdapter();
+    const dispose = adapter.open(accessor);
+
+    // store -> doc
+    adapter.write({ added: [{ id: 'a', value: '1' }] });
+    expect(elementMap(accessor).a).toEqual({ id: 'a', value: '1' });
+
+    // doc -> store (external mutation)
+    mutateMap(accessor, (map) => {
+      map.b = { id: 'b', value: '2' };
+    });
+    expect(adapter.store.get('b')).toEqual({ id: 'b', value: '2' });
+
+    // dispose stops propagation
+    dispose();
+    mutateMap(accessor, (map) => {
+      map.c = { id: 'c', value: '3' };
+    });
+    expect(adapter.store.has('c')).toBe(false);
+  });
+});

package/src/store-adapter.ts

@@ -0,0 +1,237 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { next as A } from '@automerge/automerge';
+
+import { invariant } from '@dxos/invariant';
+import { log } from '@dxos/log';
+import { isNonNullable } from '@dxos/util';
+
+import * as Doc from './Doc';
+import { decode as defaultDecode, encode as defaultEncode, getDeep, rebasePath } from './record';
+
+export type BaseElement = { id: string };
+
+/**
+ * Batch of element changes.
+ */
+export type Batch<Element extends BaseElement> = {
+  added?: Element[];
+  updated?: Element[];
+  deleted?: Element['id'][];
+};
+
+/**
+ * Accumulates pending local mutations.
+ */
+export class Modified<Element extends BaseElement> {
+  readonly added = new Map<Element['id'], Element>();
+  readonly updated = new Map<Element['id'], Element>();
+  readonly deleted = new Set<Element['id']>();
+
+  batch(): Batch<Element> {
+    return {
+      added: Array.from(this.added.values()),
+      updated: Array.from(this.updated.values()),
+      deleted: Array.from(this.deleted.values()),
+    };
+  }
+
+  clear(): void {
+    this.added.clear();
+    this.updated.clear();
+    this.deleted.clear();
+  }
+}
+
+export type StoreAdapterOptions = {
+  readonly?: boolean;
+  /** Encode a model value to an Automerge record (defaults to a structural codec). */
+  encode?: (value: any) => any;
+  /** Decode an Automerge record to a model value. */
+  decode?: (value: any) => any;
+};
+
+/**
+ * Two-way sync between an external store and an id-keyed `Record<string, Element>` held at an
+ * accessor's path within an Automerge document. Subclasses bind a concrete store (e.g. tldraw,
+ * excalidraw) by implementing {@link getElements} / {@link onUpdate} and registering store
+ * listeners in {@link onOpen}.
+ */
+// TODO(burdon): Make element id key configurable.
+export abstract class AbstractStoreAdapter<Element extends BaseElement> {
+  #accessor?: Doc.Accessor<any>;
+  #lastHeads?: A.Heads;
+  #cleanup?: () => void;
+  readonly #readonly: boolean;
+  readonly #encode: (value: any) => any;
+  readonly #decode: (value: any) => any;
+
+  constructor(options: StoreAdapterOptions = {}) {
+    this.#readonly = options.readonly ?? false;
+    this.#encode = options.encode ?? defaultEncode;
+    this.#decode = options.decode ?? defaultDecode;
+  }
+
+  get isOpen(): boolean {
+    return !!this.#accessor;
+  }
+
+  get readonly(): boolean {
+    return this.#readonly;
+  }
+
+  /**
+   * Binds the adapter to the element map at `accessor.path`. Returns a dispose function; calling it
+   * (or {@link close}) tears down the subscription and any listeners registered in {@link onOpen}.
+   */
+  open(accessor: Doc.Accessor<any>): () => void {
+    invariant(accessor.path.length);
+    if (this.isOpen) {
+      this.close();
+    }
+
+    log('opening...', { path: accessor.path });
+    this.#accessor = accessor;
+    const onOpenCleanup = this.onOpen();
+
+    // Seed the document from the store, or hydrate the store from the document.
+    {
+      const map: Record<string, Element> = getDeep(accessor.handle.doc(), accessor.path) ?? {};
+      const records = Object.values(map);
+      if (records.length === 0) {
+        accessor.handle.change((doc) => {
+          const map: Record<string, Element> = getDeep(doc, accessor.path, true);
+          for (const record of this.getElements()) {
+            map[record.id] = this.#encode(record);
+          }
+        });
+      } else {
+        this.onUpdate({ updated: records.map((record) => this.#decode(record)) });
+      }
+    }
+
+    // Baseline heads after seeding so the first change diffs only post-open mutations (not full history).
+    this.#lastHeads = A.getHeads(accessor.handle.doc());
+
+    // Propagate document mutations (local and remote) into the store.
+    const updateModel = () => {
+      const doc = accessor.handle.doc();
+      const map: Record<string, Element> = getDeep(doc, accessor.path);
+
+      const updated = new Set<Element['id']>();
+      const deleted = new Set<Element['id']>();
+
+      const currentHeads = A.getHeads(doc);
+      const diff = A.equals(this.#lastHeads, currentHeads) ? [] : A.diff(doc, this.#lastHeads ?? [], currentHeads);
+      diff.forEach((patch) => {
+        const relativePath = rebasePath(patch.path, accessor.path);
+        if (!relativePath) {
+          return;
+        }
+
+        // A patch on the map root (e.g. initial assignment) touches every element.
+        if (relativePath.length === 0) {
+          for (const id of Object.keys(map)) {
+            updated.add(id as Element['id']);
+          }
+          return;
+        }
+
+        switch (patch.action) {
+          case 'del': {
+            if (relativePath.length === 1) {
+              deleted.add(relativePath[0] as Element['id']);
+              break;
+            }
+          }
+          // eslint-disable-next-line no-fallthrough
+          case 'put':
+          case 'insert':
+          case 'inc':
+          case 'splice': {
+            updated.add(relativePath[0] as Element['id']);
+            break;
+          }
+          default:
+            log.warn('did not process patch', { patch, path: accessor.path });
+        }
+      });
+
+      if (updated.size || deleted.size) {
+        this.onUpdate({
+          updated: Array.from(updated)
+            .map((id) => this.#decode(map[id]))
+            .filter(isNonNullable), // Elements modified then eventually removed.
+          deleted: Array.from(deleted),
+        });
+      }
+
+      this.#lastHeads = currentHeads;
+    };
+
+    accessor.handle.addListener('change', updateModel);
+    this.#cleanup = () => {
+      accessor.handle.removeListener('change', updateModel);
+      onOpenCleanup?.();
+    };
+
+    log('open');
+    return () => this.close();
+  }
+
+  close(): void {
+    if (!this.isOpen) {
+      return;
+    }
+
+    log('closing...');
+    this.onClose();
+    this.#cleanup?.();
+    this.#cleanup = undefined;
+    this.#accessor = undefined;
+    this.#lastHeads = undefined;
+    log('closed');
+  }
+
+  /**
+   * Writes a batch of element changes to the document.
+   */
+  protected updateDatabase(batch: Batch<Element>): void {
+    invariant(this.isOpen);
+    if (this.#readonly) {
+      log.warn('attempting to update read-only store');
+      return;
+    }
+
+    const accessor = this.#accessor!;
+    accessor.handle.change((doc) => {
+      const map: Record<string, Element> = getDeep(doc, accessor.path, true);
+      this.#removeDeleted(batch, batch.added)?.forEach((element) => (map[element.id] = this.#encode(element)));
+      this.#removeDeleted(batch, batch.updated)?.forEach((element) => (map[element.id] = this.#encode(element)));
+      batch.deleted?.forEach((id) => delete map[id]);
+    });
+  }
+
+  #removeDeleted(batch: Batch<Element>, elements?: Element[]): Element[] | undefined {
+    return batch.deleted ? elements?.filter((element) => !batch.deleted!.includes(element.id)) : elements;
+  }
+
+  /**
+   * Returns all elements currently in the store.
+   */
+  abstract getElements(): readonly Element[];
+
+  /**
+   * Applies document changes to the store.
+   */
+  protected abstract onUpdate(batch: Batch<Element>): void;
+
+  /**
+   * Called when the adapter opens; return a cleanup function to run on close (e.g. to remove store listeners).
+   */
+  protected onOpen(): (() => void) | void {}
+
+  protected onClose(): void {}
+}