@kyneta/yjs-schema 1.6.1 → 1.8.0

package/src/substrate.ts

@@ -1,21 +1,38 @@
 // substrate — YjsSubstrate implementation.
 //
 // Implements Substrate<YjsVersion> with:
-// - Imperative local writes (prepare accumulates, onFlush applies in transact)
-// - Persistent observeDeep event bridge for external changes
-// - Transaction-origin filter (`KYNETA_ORIGIN`) to ignore our own writes.
+// - Imperative-eager local writes: `prepare` advances both the shadow σ
+//   AND the native Y.Doc tree λ inside the ambient `Y.transact` opened
+//   by `runBatch`. The projection law `σ ≡ Π(λ)` holds at every prepare
+//   boundary — re-entrant subscribers reading either σ (via the Reader)
+//   or λ (via `unwrap`) see a coherent state.
+// - `runBatch(body)` opens one `Y.transact(doc, body, options?.origin)` per
+//   outermost logical action. Yjs's native transact nesting collapses
+//   inner re-entrant transacts into the outer one for free — no depth
+//   counter needed (unlike Loro). External `observeDeep` consumers see
+//   exactly one batched event per outermost `change(doc, fn)`.
+// - JSON-boundary writes (struct.json/list.json/record.json subtrees)
+//   are buffered in a per-target-key coalescer and flushed in
+//   `afterBatch`. Non-boundary writes are applied directly to λ via
+//   `applyChangeToYjs`.
+// - `afterBatch` flushes the json-boundary coalescer on local writes
+//   and re-materialises σ from λ on replay.
+// - Persistent observeDeep event bridge for external changes.
+// - Per-transaction meta mark (`KYNETA_MARK`) inscribed from inside the
+//   transact body to ignore our own writes; survives Yjs's nested-transact
+//   collapse so external wrapping is handled correctly.
 //
 // The event bridge contract: wrapping a Y.Doc in a kyneta substrate
 // means subscribing to the kyneta doc observes ALL mutations to the
 // underlying Y.Doc, regardless of source (local kyneta writes,
 // merge, external Y.applyUpdate, external raw Yjs API mutations).
 //
-// `prepare` and `onFlush` accept `BatchOptions` and branch on
+// `prepare` and `afterBatch` accept `BatchOptions` and branch on
 // `options?.replay`. The event bridge constructs the replay batch via
 // `executeBatch(ctx, ops, { origin, replay: true })`; substrate-side
 // work (transact, write) is skipped when `replay` is true because the
 // native Y.Doc already absorbed the change. This makes `prepare` and
-// `onFlush` total functions of their declared inputs — no hidden
+// `afterBatch` total functions of their declared inputs — no hidden
 // ambient state for the substrate-write decision. Context: jj:qpultxsw.
 //
 // Identity-keying: when a SchemaBinding is provided, all Y.Map key
@@ -26,9 +43,11 @@ import type {
   BatchOptions,
   ChangeBase,
   Path,
+  PlainState,
   PositionCapable,
   ProductSchema,
   Reader,
+  RecordInverseFn,
   Replica,
   ReplicaFactory,
   SchemaBinding,
@@ -41,25 +60,41 @@ import type {
   WritableContext,
 } from "@kyneta/schema"
 import {
+  applyChange,
   BACKING_DOC,
   buildWritableContext,
+  deepClonePreState,
   deriveSchemaBinding,
   executeBatch,
+  findJsonBoundary,
+  invert,
   KIND,
+  plainReader,
+  RECORD_INVERSE,
+  syncShadow,
 } from "@kyneta/schema"
 import * as Y from "yjs"
 import { applyChangeToYjs, eventsToOps } from "./change-mapping.js"
+import { materializeYjsShadow } from "./materialize.js"
 import { ensureContainers } from "./populate.js"
 import { toYjsAssoc, YjsPosition } from "./position.js"
-import { yjsReader } from "./reader.js"
 import { YjsVersion } from "./version.js"
 import { resolveYjsType } from "./yjs-resolve.js"
 
 // ---------------------------------------------------------------------------
-// Origin tag — used to suppress echo from our own transactions
+// Own-commit discriminator
 // ---------------------------------------------------------------------------
 
-const KYNETA_ORIGIN = "kyneta-prepare"
+// Own-commit discriminator. We mark `transaction.meta` from inside
+// the transact body — the mark travels with the Transaction object
+// that Yjs hands to observeDeep, regardless of `transaction.origin`.
+// This frees the user-facing `origin` slot for `options.origin`
+// round-trip and correctly handles the case where external code
+// wraps `change(doc, fn)` in its own `Y.transact` (Yjs's nested-
+// transact collapse delivers the SAME Transaction object to both
+// outer and inner callbacks; verified by probe — see TECHNICAL.md
+// "Why transaction.meta mark").
+const KYNETA_MARK = Symbol("kyneta:own-commit")
 
 // ---------------------------------------------------------------------------
 // createYjsSubstrate — wrap a user-provided Y.Doc
@@ -91,8 +126,20 @@ export function createYjsSubstrate(
 ): Substrate<YjsVersion> {
   // --- Closure-scoped state ---
 
-  // Accumulated changes from prepare(), drained by onFlush().
-  const pendingChanges: Array<{ path: Path; change: ChangeBase }> = []
+  // JSON-boundary coalescing buffer. Keyed by the target Y.Map and the
+  // boundary key — repeated writes inside the same struct.json /
+  // record.json subtree overwrite the entry with the latest σ value
+  // before `afterBatch` flushes it back into λ as a single
+  // `target.set(key, value)`. Non-boundary writes bypass the buffer
+  // entirely — they go straight to `applyChangeToYjs` in prepare.
+  const jsonBoundaryBuffer = new Map<
+    string,
+    {
+      target: Y.Map<unknown> | Y.Array<unknown>
+      key: string | number
+      value: unknown
+    }
+  >()
 
   // Stashed origin from merge for the event bridge to pick up.
   let pendingMergeOrigin: string | undefined
@@ -100,19 +147,102 @@ export function createYjsSubstrate(
   // Lazy-built WritableContext (same pattern as PlainSubstrate / LoroSubstrate).
   let cachedCtx: WritableContext | undefined
 
-  // Incremental delete set tracking.
-  // Initialized once from the struct store (O(n)), then maintained
-  // incrementally by merging each transaction's deleteSet (O(delta)).
-  // Note: DeleteSet class isn't exported from yjs's public entry point,
-  // so we infer the type from createDeleteSet's return type.
-  let accumulatedDs: ReturnType<typeof Y.createDeleteSet> =
-    Y.createDeleteSetFromStructStore(doc.store)
-
   // The root Y.Map — all schema fields are children of this single map.
   const rootMap = doc.getMap("root")
 
-  // The Reader — live view over the Yjs shared type tree.
-  const reader: Reader = yjsReader(doc, schema, binding)
+  // The shadow — a plain JS object materialized from the Y.Doc.
+  // Kept in sync by applyChange() in prepare().
+  const shadow: PlainState = materializeYjsShadow(doc, schema, binding)
+  const reader: Reader = plainReader(shadow)
+
+  // --- Coalescer helpers ---
+
+  /**
+   * Compute the identity-aware boundary key (or numeric index) for a
+   * json-boundary write at `prefixLength`. Mirrors the Loro substrate's
+   * `boundaryKey`; field segments inside a bound product get the
+   * identity hash, others pass through raw.
+   */
+  function boundaryKey(path: Path, prefixLength: number): string | number {
+    const seg = path.segments[prefixLength]!
+    if (seg.role === "field" && binding) {
+      const absPath = path.segments
+        .slice(0, prefixLength + 1)
+        .filter(s => s.role === "field")
+        .map(s => s.resolve() as string)
+        .join(".")
+      const identity = binding.forward.get(absPath) as string | undefined
+      if (identity) return identity
+    }
+    return seg.resolve() as string | number
+  }
+
+  /**
+   * Buffer a json-boundary write. The boundary value is the entire σ
+   * subtree at the boundary path — already updated by the preceding
+   * `applyChange(shadow, ...)`. Subsequent writes inside the same
+   * subtree overwrite this entry (last-write-wins by σ snapshot).
+   *
+   * Returns silently when the parent container can't be resolved
+   * (root-level json fields land in `rootMap` directly — Yjs's
+   * root is the rootMap, so the parentResolved is `rootMap`).
+   */
+  function stageJsonBoundaryWrite(path: Path, prefixLength: number): void {
+    const parentPath = path.slice(0, prefixLength)
+    const { resolved: parent } = resolveYjsType(
+      rootMap,
+      schema,
+      parentPath,
+      binding,
+    )
+    const boundaryPath = path.slice(0, prefixLength + 1)
+    const value = boundaryPath.read(shadow)
+    const key = boundaryKey(path, prefixLength)
+
+    // The target can be either a Y.Map (struct field, record entry,
+    // or rootMap) or a Y.Array (list/movable item). Both expose a
+    // shape we can stash and flush in `afterBatch`.
+    let target: Y.Map<unknown> | Y.Array<unknown>
+    if (parent instanceof Y.Map) {
+      target = parent
+    } else if (parent instanceof Y.Array) {
+      target = parent
+    } else {
+      throw new Error(
+        `yjs substrate: json-boundary write to unsupported parent type at path ${path.format()}`,
+      )
+    }
+
+    // Use the Yjs shared-type's stable identity for the buffer key
+    // when available; fall back to a unique sentinel for the
+    // ultra-rare case where `_item` is undefined (freshly-created
+    // shared types before they're attached). Combine with key/index
+    // for a unique slot — repeat writes to the same slot overwrite.
+    const targetId = `${(target as any)._item?.id?.client ?? "root"}:${(target as any)._item?.id?.clock ?? "root"}`
+    const slot = `${targetId}/${String(key)}`
+    jsonBoundaryBuffer.set(slot, { target, key, value })
+  }
+
+  /**
+   * Drain the json-boundary buffer into λ. Called from `afterBatch`
+   * inside the ambient `Y.transact` opened by `runBatch`. Each entry
+   * is applied as `target.set(key, value)` for Y.Map parents or as a
+   * delete+insert for Y.Array parents (Yjs Arrays don't have a
+   * `set(index, value)` primitive — replace = delete one + insert one).
+   */
+  function flushJsonBoundaryBuffer(): void {
+    if (jsonBoundaryBuffer.size === 0) return
+    for (const { target, key, value } of jsonBoundaryBuffer.values()) {
+      if (target instanceof Y.Map) {
+        target.set(String(key), value)
+      } else {
+        const index = key as number
+        target.delete(index, 1)
+        target.insert(index, [value])
+      }
+    }
+    jsonBoundaryBuffer.clear()
+  }
 
   // --- Substrate object ---
 
@@ -122,86 +252,142 @@ export function createYjsSubstrate(
     reader: reader,
 
     prepare(path: Path, change: ChangeBase, options?: BatchOptions): void {
-      if (options?.replay) {
-        // Yjs already has these ops (the bridge is replaying them
-        // through kyneta solely so the changefeed layer can deliver
-        // notifications — wrappedPrepare buffered the op upstream).
+      // Replay writes: λ has already absorbed these ops via
+      // Y.applyUpdate at the event-bridge call site; skip σ/λ
+      // advance — afterBatch(replay) rebuilds σ from λ in one
+      // Π pass.
+      if (options?.replay) return
+
+      // Inverse recording under the normal handler. Capture σ at the
+      // target path before applyChange mutates the shadow; the
+      // recording closure pushes onto the active runBatch frame's
+      // stack. Skipped under the undo-replay handler (compensating).
+      const record = (
+        options as
+          | (BatchOptions & { [RECORD_INVERSE]?: RecordInverseFn })
+          | undefined
+      )?.[RECORD_INVERSE]
+      if (record && !options?.compensating) {
+        const pre = deepClonePreState(path.read(shadow))
+        const inverse = invert(pre, change)
+        record(path, inverse)
+      }
+
+      // Local write — σ advances eagerly. CRDT-side writes happen
+      // inside the ambient Y.transact opened by runBatch (the
+      // substrate's `runBatch` wraps `executeBatch`'s prepare-loop +
+      // flush).
+      applyChange(shadow, path, change)
+
+      // JSON-boundary write: stage a full-value write at the
+      // boundary segment of the parent container. Coalesces with
+      // repeated writes inside the same subtree (last σ snapshot
+      // wins) and lands in λ on `afterBatch` flush.
+      const boundary = findJsonBoundary(schema, path, binding)
+      if (boundary !== null) {
+        stageJsonBoundaryWrite(path, boundary.prefixLength)
         return
       }
-      // Mutations happen in onFlush inside a single Yjs transaction.
-      pendingChanges.push({ path, change })
+
+      // Non-boundary write: imperatively apply to λ inside the
+      // ambient Y.transact. The KYNETA_ORIGIN tag lets the
+      // observeDeep bridge below recognise and skip the events we
+      // generate here, so the changefeed isn't fired twice.
+      applyChangeToYjs(rootMap, schema, path, change, binding)
     },
 
-    onFlush(options?: BatchOptions): void {
+    afterBatch(options?: BatchOptions): void {
       if (options?.replay) {
-        // Yjs already committed; wrappedFlush still delivers
-        // notifications upstream.
+        // CRDT merge is a lattice join — re-materialise σ from λ in
+        // one Π pass instead of replaying ops incrementally.
+        syncShadow(shadow, materializeYjsShadow(doc, schema, binding))
         return
       }
-      if (pendingChanges.length === 0) return
-      // The KYNETA_ORIGIN tag lets the observeDeep bridge below
-      // recognise and skip the events we generate here, so the
-      // changefeed isn't fired twice for the same write.
-      doc.transact(() => {
-        for (const { path, change } of pendingChanges) {
-          applyChangeToYjs(rootMap, schema, path, change, binding)
-        }
-      }, KYNETA_ORIGIN)
-      pendingChanges.length = 0
+      // Local write: drain the json-boundary coalescer. Runs inside
+      // the ambient Y.transact from `runBatch`; the transact closes
+      // when `runBatch`'s body returns, emitting one batched
+      // observeDeep event for the whole logical action.
+      flushJsonBoundaryBuffer()
+    },
+
+    runBatch(work: () => void, options?: BatchOptions): void {
+      // Yjs's native transact nesting collapses inner re-entrant
+      // transacts into the outermost — exactly the "one batched
+      // event per outermost logical action" semantic we want. No
+      // depth counter needed.
+      //
+      // We mark the transaction via `tr.meta.set` inside the transact body.
+      // The mark lives on per-transaction meta, orthogonal to origin.
+      // The app-level `options?.origin` flows directly to `transaction.origin`
+      // and round-trips to the changefeed layer.
+      doc.transact(tr => {
+        tr.meta.set(KYNETA_MARK, true)
+        work()
+      }, options?.origin)
     },
 
     context(): WritableContext {
       if (!cachedCtx) {
-        cachedCtx = buildWritableContext(substrate)
-        // Attach nativeResolver — used by interpretImpl to set [NATIVE]
-        // on every ref. The resolver maps schema positions to Yjs shared types.
-        ;(cachedCtx as any).nativeResolver = (
-          nodeSchema: SchemaNode,
-          path: { segments: readonly unknown[] },
-        ) => {
-          if (path.segments.length === 0) return doc
-          if (nodeSchema[KIND] === "scalar" || nodeSchema[KIND] === "sum")
-            return undefined
-          return resolveYjsType(rootMap, schema, path as any, binding).resolved
-        }
-        ;(cachedCtx as any).positionResolver = (
-          _nodeSchema: unknown,
-          path: { segments: readonly unknown[] },
-        ) => {
-          return {
-            createPosition(index: number, side: Side) {
-              // Resolve path to the Y.Text shared type
-              const { resolved: ytype } = resolveYjsType(
-                rootMap,
-                schema,
-                path as any,
-                binding,
-              )
-              if (!(ytype instanceof Y.Text)) {
-                throw new Error(
-                  `positionResolver: path does not resolve to a Y.Text`,
+        cachedCtx = buildWritableContext(substrate, {
+          nativeResolver: (
+            nodeSchema: SchemaNode,
+            path: { segments: readonly unknown[] },
+          ) => {
+            if (path.segments.length === 0) return doc
+            if (nodeSchema[KIND] === "scalar" || nodeSchema[KIND] === "sum")
+              return undefined
+            return resolveYjsType(rootMap, schema, path as any, binding)
+              .resolved
+          },
+          positionResolver: (
+            _nodeSchema: unknown,
+            path: { segments: readonly unknown[] },
+          ) => {
+            return {
+              createPosition(index: number, side: Side) {
+                // Resolve path to the Y.Text shared type
+                const { resolved: ytype } = resolveYjsType(
+                  rootMap,
+                  schema,
+                  path as any,
+                  binding,
+                )
+                if (!(ytype instanceof Y.Text)) {
+                  throw new Error(
+                    `positionResolver: path does not resolve to a Y.Text`,
+                  )
+                }
+                const assoc = toYjsAssoc(side)
+                const rpos = Y.createRelativePositionFromTypeIndex(
+                  ytype,
+                  index,
+                  assoc,
                 )
-              }
-              const assoc = toYjsAssoc(side)
-              const rpos = Y.createRelativePositionFromTypeIndex(
-                ytype,
-                index,
-                assoc,
-              )
-              return new YjsPosition(rpos, doc)
-            },
-            decodePosition(bytes: Uint8Array) {
-              const rpos = Y.decodeRelativePosition(bytes)
-              return new YjsPosition(rpos, doc)
-            },
-          } satisfies PositionCapable
-        }
+                return new YjsPosition(rpos, doc)
+              },
+              decodePosition(bytes: Uint8Array) {
+                const rpos = Y.decodeRelativePosition(bytes)
+                return new YjsPosition(rpos, doc)
+              },
+            } satisfies PositionCapable
+          },
+        })
       }
       return cachedCtx
     },
 
     version(): YjsVersion {
-      return YjsVersion.fromDeleteSet(doc, accumulatedDs)
+      // Derive the deleteSet from the live struct store on every read.
+      // Eager-prepare delivers notifications *inside* the ambient
+      // `Y.transact` opened by `runBatch`, so `afterTransaction` fires
+      // AFTER the changefeed's notify pipeline. Computing from the
+      // store picks up in-progress deletes too, so the exchange's
+      // auto-subscribe sees a version that already reflects the
+      // just-applied mutation.
+      return YjsVersion.fromDeleteSet(
+        doc,
+        Y.createDeleteSetFromStructStore(doc.store),
+      )
     },
 
     baseVersion(): YjsVersion {
@@ -259,8 +445,11 @@ export function createYjsSubstrate(
   // --- Event bridge (registered once at construction) ---
 
   rootMap.observeDeep((events, transaction) => {
-    // Ignore our own transactions (changefeed already captured via wrappedPrepare)
-    if (transaction.origin === KYNETA_ORIGIN) {
+    // Own-commit discriminator: kyneta's runBatch marks the transaction
+    // via `tr.meta.set` inside the transact body. The mark survives Yjs's
+    // nested-transact collapse, so external code wrapping `change()` in
+    // its own Y.transact is correctly classified as own.
+    if (transaction.meta.get(KYNETA_MARK)) {
       return
     }
 
@@ -270,12 +459,6 @@ export function createYjsSubstrate(
       return
     }
 
-    // Update accumulated delete set BEFORE executeBatch, so version()
-    // reflects deletes when notifyLocalChange fires from the changefeed.
-    if (transaction.deleteSet.clients.size > 0) {
-      accumulatedDs = Y.mergeDeleteSets([accumulatedDs, transaction.deleteSet])
-    }
-
     // Determine origin: prefer stashed kyneta origin (from merge),
     // fall back to the transaction's origin if it's a string.
     const origin =
@@ -285,26 +468,12 @@ export function createYjsSubstrate(
     // Lazily ensure the context is built
     const ctx = substrate.context()
 
-    // `replay: true` tells substrate.prepare/onFlush to skip native-side
+    // `replay: true` tells substrate.prepare/afterBatch to skip native-side
     // work (Yjs has already absorbed these ops via Y.applyUpdate) and
     // surfaces on the Changeset for downstream filters (exchange echo).
     executeBatch(ctx, ops, { origin, replay: true })
   })
 
-  // For local mutations (KYNETA_ORIGIN): the observeDeep handler returns
-  // early, so we merge the delete set via afterTransaction instead.
-  // afterTransaction fires inside doc.transact() — before onFlush returns
-  // — so accumulatedDs is up to date when the changefeed's
-  // deliverNotifications → notifyLocalChange → version() fires.
-  doc.on("afterTransaction", (transaction: Y.Transaction) => {
-    if (
-      transaction.origin === KYNETA_ORIGIN &&
-      transaction.deleteSet.clients.size > 0
-    ) {
-      accumulatedDs = Y.mergeDeleteSets([accumulatedDs, transaction.deleteSet])
-    }
-  })
-
   return substrate as Substrate<YjsVersion>
 }
 
@@ -474,17 +643,14 @@ export const yjsSubstrateFactory: SubstrateFactory<YjsVersion> = {
     const doc = (replica as any)[BACKING_DOC] as Y.Doc
     const binding = trivialBinding(schema)
     // No identity injection for the standalone factory (no peerId).
-    // Conditional ensureContainers: skip fields that already exist
-    // from hydrated state.
-    ensureContainers(doc, schema, true, binding)
+    ensureContainers(doc, schema, binding)
     return createYjsSubstrate(doc, schema, binding)
   },
 
   create(schema: SchemaNode): Substrate<YjsVersion> {
-    // Fresh doc — unconditional ensureContainers (nothing to conflict with).
     const doc = new Y.Doc()
     const binding = trivialBinding(schema)
-    ensureContainers(doc, schema, false, binding)
+    ensureContainers(doc, schema, binding)
     return createYjsSubstrate(doc, schema, binding)
   },
 

package/src/yjs-extract.ts

@@ -0,0 +1,52 @@
+// yjs-extract — shared value-extraction helpers for Yjs shared types.
+//
+// These functions are used by both the reader (yjsReader) and the
+// materialize interpreter to convert Yjs shared types into plain values.
+
+import type { RichTextDelta, RichTextSpan } from "@kyneta/schema"
+import * as Y from "yjs"
+
+/**
+ * Extract a plain value from a Yjs shared type or return a plain value as-is.
+ *
+ * - Y.Text → `.toJSON()` (string)
+ * - Y.Map → `.toJSON()` (plain object snapshot — for product/map reads)
+ * - Y.Array → `.toJSON()` (plain array snapshot)
+ * - Plain values (string, number, boolean, null) → returned as-is
+ */
+export function extractValue(resolved: unknown): unknown {
+  if (resolved instanceof Y.Text) {
+    return resolved.toJSON()
+  }
+  if (resolved instanceof Y.Map) {
+    return resolved.toJSON()
+  }
+  if (resolved instanceof Y.Array) {
+    return resolved.toJSON()
+  }
+  // Plain scalar value (string, number, boolean, null, etc.)
+  return resolved
+}
+
+/**
+ * Convert a Y.Text's delta (Quill format) to a kyneta RichTextDelta.
+ *
+ * Yjs `.toDelta()` returns `{ insert: string, attributes?: Record<string, any> }[]`.
+ * Kyneta RichTextDelta is `{ text: string, marks?: MarkMap }[]`.
+ */
+export function yTextToRichTextDelta(ytext: Y.Text): RichTextDelta {
+  const delta = ytext.toDelta() as Array<{
+    insert: string
+    attributes?: Record<string, unknown>
+  }>
+  const spans: RichTextSpan[] = []
+  for (const d of delta) {
+    if (typeof d.insert !== "string") continue
+    const span: RichTextSpan =
+      d.attributes && Object.keys(d.attributes).length > 0
+        ? { text: d.insert, marks: d.attributes }
+        : { text: d.insert }
+    spans.push(span)
+  }
+  return spans
+}