@kyneta/yjs-schema 1.6.0 → 1.7.0

package/src/substrate.ts

@@ -3,20 +3,30 @@
 // Implements Substrate<YjsVersion> with:
 // - Imperative local writes (prepare accumulates, onFlush applies in transact)
 // - Persistent observeDeep event bridge for external changes
-// - Single re-entrancy guard + transaction.origin check
+// - Transaction-origin filter (`KYNETA_ORIGIN`) to ignore our own writes.
 //
 // The event bridge contract: wrapping a Y.Doc in a kyneta substrate
 // means subscribing to the kyneta doc observes ALL mutations to the
 // underlying Y.Doc, regardless of source (local kyneta writes,
 // merge, external Y.applyUpdate, external raw Yjs API mutations).
 //
+// `prepare` and `onFlush` accept `BatchOptions` and branch on
+// `options?.replay`. The event bridge constructs the replay batch via
+// `executeBatch(ctx, ops, { origin, replay: true })`; substrate-side
+// work (transact, write) is skipped when `replay` is true because the
+// native Y.Doc already absorbed the change. This makes `prepare` and
+// `onFlush` total functions of their declared inputs — no hidden
+// ambient state for the substrate-write decision. Context: jj:qpultxsw.
+//
 // Identity-keying: when a SchemaBinding is provided, all Y.Map key
 // lookups and writes use the identity hash instead of the field name.
 // The binding is threaded to the reader, event bridge, and write path.
 
 import type {
+  BatchOptions,
   ChangeBase,
   Path,
+  PlainState,
   PositionCapable,
   ProductSchema,
   Reader,
@@ -32,17 +42,19 @@ import type {
   WritableContext,
 } from "@kyneta/schema"
 import {
+  applyChange,
   BACKING_DOC,
   buildWritableContext,
   deriveSchemaBinding,
   executeBatch,
   KIND,
+  plainReader,
 } from "@kyneta/schema"
 import * as Y from "yjs"
 import { applyChangeToYjs, eventsToOps } from "./change-mapping.js"
+import { materializeYjsShadow } from "./materialize.js"
 import { ensureContainers } from "./populate.js"
 import { toYjsAssoc, YjsPosition } from "./position.js"
-import { yjsReader } from "./reader.js"
 import { YjsVersion } from "./version.js"
 import { resolveYjsType } from "./yjs-resolve.js"
 
@@ -85,12 +97,6 @@ export function createYjsSubstrate(
   // Accumulated changes from prepare(), drained by onFlush().
   const pendingChanges: Array<{ path: Path; change: ChangeBase }> = []
 
-  // Re-entrancy guard: set true around our doc.transact() in onFlush
-  // AND around executeBatch in the event bridge. When true, prepare()
-  // skips Yjs-side work (changes are already applied by Yjs or about
-  // to be), and onFlush() skips transact/commit.
-  let inOurTransaction = false
-
   // Stashed origin from merge for the event bridge to pick up.
   let pendingMergeOrigin: string | undefined
 
@@ -108,8 +114,10 @@ export function createYjsSubstrate(
   // The root Y.Map — all schema fields are children of this single map.
   const rootMap = doc.getMap("root")
 
-  // The Reader — live view over the Yjs shared type tree.
-  const reader: Reader = yjsReader(doc, schema, binding)
+  // The shadow — a plain JS object materialized from the Y.Doc.
+  // Kept in sync by applyChange() in prepare().
+  const shadow: PlainState = materializeYjsShadow(doc, schema, binding)
+  const reader: Reader = plainReader(shadow)
 
   // --- Substrate object ---
 
@@ -118,34 +126,44 @@ export function createYjsSubstrate(
 
     reader: reader,
 
-    prepare(path: Path, change: ChangeBase): void {
-      if (!inOurTransaction) {
-        // Local write: accumulate for flush.
-        // No Yjs side effects — mutations happen at flush time.
-        pendingChanges.push({ path, change })
+    prepare(path: Path, change: ChangeBase, options?: BatchOptions): void {
+      // Local writes: apply eagerly to the shadow so reads are
+      // immediately consistent. Replay writes: skip — the shadow
+      // will be re-materialized from the Y.Doc in onFlush(replay).
+      if (!options?.replay) {
+        applyChange(shadow, path, change)
+      }
+
+      if (options?.replay) {
+        return
       }
-      // During event handler replay: no-op on Yjs side.
-      // wrappedPrepare (changefeed layer) still buffers the op.
+      pendingChanges.push({ path, change })
     },
 
-    onFlush(_origin?: string): void {
-      if (!inOurTransaction && pendingChanges.length > 0) {
-        // Local write: apply accumulated changes within a single
-        // Yjs transaction tagged with our origin for echo suppression.
-        inOurTransaction = true
-        try {
-          doc.transact(() => {
-            for (const { path, change } of pendingChanges) {
-              applyChangeToYjs(rootMap, schema, path, change, binding)
-            }
-          }, KYNETA_ORIGIN)
-          pendingChanges.length = 0
-        } finally {
-          inOurTransaction = false
+    onFlush(options?: BatchOptions): void {
+      if (options?.replay) {
+        // Re-materialize shadow from the Y.Doc (already committed).
+        const fresh = materializeYjsShadow(doc, schema, binding)
+        for (const key of Object.keys(fresh)) {
+          shadow[key] = fresh[key]
         }
+        for (const key of Object.keys(shadow)) {
+          if (!(key in fresh)) {
+            delete shadow[key]
+          }
+        }
+        return
       }
-      // During event handler replay: no-op on Yjs side.
-      // wrappedFlush (changefeed layer) still delivers notifications.
+      if (pendingChanges.length === 0) return
+      // The KYNETA_ORIGIN tag lets the observeDeep bridge below
+      // recognise and skip the events we generate here, so the
+      // changefeed isn't fired twice for the same write.
+      doc.transact(() => {
+        for (const { path, change } of pendingChanges) {
+          applyChangeToYjs(rootMap, schema, path, change, binding)
+        }
+      }, KYNETA_ORIGIN)
+      pendingChanges.length = 0
     },
 
     context(): WritableContext {
@@ -232,7 +250,7 @@ export function createYjsSubstrate(
       }
     },
 
-    merge(payload: SubstratePayload, origin?: string): void {
+    merge(payload: SubstratePayload, options?: BatchOptions): void {
       if (
         payload.encoding !== "binary" ||
         !(payload.data instanceof Uint8Array)
@@ -243,14 +261,14 @@ export function createYjsSubstrate(
         )
       }
       // Stash origin for the event bridge to pick up
-      pendingMergeOrigin = origin
+      pendingMergeOrigin = options?.origin
       try {
-        Y.applyUpdate(doc, payload.data, origin ?? "remote")
+        Y.applyUpdate(doc, payload.data, options?.origin ?? "remote")
       } finally {
         pendingMergeOrigin = undefined
       }
       // That's it — the observeDeep handler bridges events to the
-      // changefeed via executeBatch.
+      // changefeed via executeBatch with `replay: true`.
     },
   }
 
@@ -283,15 +301,10 @@ export function createYjsSubstrate(
     // Lazily ensure the context is built
     const ctx = substrate.context()
 
-    // Feed through executeBatch for changefeed delivery.
-    // The inOurTransaction guard prevents prepare/onFlush from doing
-    // Yjs-side work — the changes are already applied by Yjs.
-    inOurTransaction = true
-    try {
-      executeBatch(ctx, ops, origin)
-    } finally {
-      inOurTransaction = false
-    }
+    // `replay: true` tells substrate.prepare/onFlush to skip native-side
+    // work (Yjs has already absorbed these ops via Y.applyUpdate) and
+    // surfaces on the Changeset for downstream filters (exchange echo).
+    executeBatch(ctx, ops, { origin, replay: true })
   })
 
   // For local mutations (KYNETA_ORIGIN): the observeDeep handler returns
@@ -417,7 +430,7 @@ export function createYjsReplica(doc: Y.Doc): Replica<YjsVersion> {
       }
     },
 
-    merge(payload: SubstratePayload, _origin?: string): void {
+    merge(payload: SubstratePayload, _options?: BatchOptions): void {
       if (
         payload.encoding !== "binary" ||
         !(payload.data instanceof Uint8Array)
@@ -477,17 +490,14 @@ export const yjsSubstrateFactory: SubstrateFactory<YjsVersion> = {
     const doc = (replica as any)[BACKING_DOC] as Y.Doc
     const binding = trivialBinding(schema)
     // No identity injection for the standalone factory (no peerId).
-    // Conditional ensureContainers: skip fields that already exist
-    // from hydrated state.
-    ensureContainers(doc, schema, true, binding)
+    ensureContainers(doc, schema, binding)
     return createYjsSubstrate(doc, schema, binding)
   },
 
   create(schema: SchemaNode): Substrate<YjsVersion> {
-    // Fresh doc — unconditional ensureContainers (nothing to conflict with).
     const doc = new Y.Doc()
     const binding = trivialBinding(schema)
-    ensureContainers(doc, schema, false, binding)
+    ensureContainers(doc, schema, binding)
     return createYjsSubstrate(doc, schema, binding)
   },
 

package/src/yjs-extract.ts

@@ -0,0 +1,52 @@
+// yjs-extract — shared value-extraction helpers for Yjs shared types.
+//
+// These functions are used by both the reader (yjsReader) and the
+// materialize interpreter to convert Yjs shared types into plain values.
+
+import type { RichTextDelta, RichTextSpan } from "@kyneta/schema"
+import * as Y from "yjs"
+
+/**
+ * Extract a plain value from a Yjs shared type or return a plain value as-is.
+ *
+ * - Y.Text → `.toJSON()` (string)
+ * - Y.Map → `.toJSON()` (plain object snapshot — for product/map reads)
+ * - Y.Array → `.toJSON()` (plain array snapshot)
+ * - Plain values (string, number, boolean, null) → returned as-is
+ */
+export function extractValue(resolved: unknown): unknown {
+  if (resolved instanceof Y.Text) {
+    return resolved.toJSON()
+  }
+  if (resolved instanceof Y.Map) {
+    return resolved.toJSON()
+  }
+  if (resolved instanceof Y.Array) {
+    return resolved.toJSON()
+  }
+  // Plain scalar value (string, number, boolean, null, etc.)
+  return resolved
+}
+
+/**
+ * Convert a Y.Text's delta (Quill format) to a kyneta RichTextDelta.
+ *
+ * Yjs `.toDelta()` returns `{ insert: string, attributes?: Record<string, any> }[]`.
+ * Kyneta RichTextDelta is `{ text: string, marks?: MarkMap }[]`.
+ */
+export function yTextToRichTextDelta(ytext: Y.Text): RichTextDelta {
+  const delta = ytext.toDelta() as Array<{
+    insert: string
+    attributes?: Record<string, unknown>
+  }>
+  const spans: RichTextSpan[] = []
+  for (const d of delta) {
+    if (typeof d.insert !== "string") continue
+    const span: RichTextSpan =
+      d.attributes && Object.keys(d.attributes).length > 0
+        ? { text: d.insert, marks: d.attributes }
+        : { text: d.insert }
+    spans.push(span)
+  }
+  return spans
+}

package/src/yjs-resolve.ts

@@ -1,35 +1,29 @@
 // yjs-resolve — Yjs-specific path resolution.
 //
-// Implements stepIntoYjs and resolveYjsType for schema-guided
-// navigation of the Yjs shared type tree.
-//
-// resolveYjsType is a left-fold over path segments, accumulating
-// (currentType, currentSchema) at each step. This mirrors how
-// resolveContainer works for Loro — but uses `instanceof` for
-// runtime type discrimination instead of Loro's `.kind()` method.
+// `stepIntoYjs` is the per-step substrate dispatch; `resolveYjsType`
+// applies the core `foldPath` primitive (from `@kyneta/schema`) around
+// it. The semantic invariants of the fold — identity-keying at
+// product-field boundaries, sum-boundary short-circuit — live in
+// `fold-path.ts`, not here.
 //
 // Root container strategy: All schema fields are children of a single
 // root `Y.Map` obtained via `doc.getMap("root")`. This root map holds
 // shared types (Y.Text, Y.Array, Y.Map) and plain values uniformly.
 // Using a single root Y.Map enables one `observeDeep` call that
 // captures all mutations with correct relative paths.
-//
-// Identity-keying: when a SchemaBinding is provided, every product-field
-// boundary uses the identity hash (from binding.forward) instead of the
-// field name as the Y.Map key. The binding is threaded through
-// resolveYjsType and stepIntoYjs.
 
-import type {
-  Path,
-  SchemaBinding,
-  Schema as SchemaNode,
-  Segment,
+import {
+  foldPath,
+  type Path,
+  type PathFoldResult,
+  type PathStepper,
+  type SchemaBinding,
+  type Schema as SchemaNode,
 } from "@kyneta/schema"
-import { advanceSchema, KIND } from "@kyneta/schema"
 import * as Y from "yjs"
 
 // ---------------------------------------------------------------------------
-// stepIntoYjs — single step of the fold
+// stepIntoYjs — per-step substrate dispatch (PathStepper for Yjs)
 // ---------------------------------------------------------------------------
 
 /**
@@ -41,15 +35,16 @@ import * as Y from "yjs"
  * - `Y.Text` → terminal (cannot step further)
  * - Plain value → terminal (return `undefined`)
  *
- * @param current - The current position (a Yjs shared type or plain value)
- * @param segment - The path segment to follow
- * @param identity - Optional identity hash to use instead of the segment's resolved value
+ * `_nextSchema` is part of the `PathStepper` contract for Loro's root
+ * dispatch but is unused here — Yjs's `instanceof` dispatch doesn't
+ * need to look ahead at the next schema kind.
  */
-export function stepIntoYjs(
-  current: unknown,
-  segment: Segment,
-  identity?: string,
-): unknown {
+export const stepIntoYjs: PathStepper = (
+  current,
+  _nextSchema,
+  segment,
+  identity,
+) => {
   const resolved = segment.resolve()
 
   if (current instanceof Y.Map) {
@@ -69,85 +64,25 @@ export function stepIntoYjs(
 }
 
 // ---------------------------------------------------------------------------
-// resolveYjsType — full path resolution via left-fold
+// resolveYjsType — full path resolution via foldPath
 // ---------------------------------------------------------------------------
 
-/**
- * Result of resolving a Yjs shared type at a path.
- *
- * Includes both the resolved Yjs value and the schema at that position,
- * enabling callers to distinguish between schema kinds that map to the
- * same Yjs type (e.g. "text" vs "richtext" both use Y.Text).
- */
-export interface ResolvedYjs {
-  readonly resolved: unknown
-  readonly schema: SchemaNode
-}
-
 /**
  * Resolve a Yjs shared type (or plain value) at the given path.
  *
- * Left-folds over path segments using `advanceSchema` for pure schema
- * descent and `stepIntoYjs` for Yjs-specific navigation.
+ * Thin wrapper around `foldPath(stepIntoYjs, ...)`. Returns the
+ * `PathFoldResult` shape from core — `{ resolved, schema }`.
  *
- * When a `binding` is provided, each step computes the absolute schema
- * path and looks up the identity hash from `binding.forward`. This
- * identity hash is used instead of the field name at every product-field
- * boundary (root and nested).
+ * When a `binding` is provided, every product-field boundary uses the
+ * identity hash from `binding.forward` instead of the field name.
  *
- * Returns both the Yjs shared type (or plain value) and the schema at
- * the terminal position. For an empty path, returns the root map and
- * root schema.
- *
- * @param rootMap - The root `Y.Map` obtained via `doc.getMap("root")`
- * @param rootSchema - The root document schema
- * @param path - The path to resolve
- * @param binding - Optional SchemaBinding for identity-keyed navigation.
+ * For an empty path, returns the root map and root schema.
  */
 export function resolveYjsType(
   rootMap: Y.Map<any>,
   rootSchema: SchemaNode,
   path: Path,
   binding?: SchemaBinding,
-): ResolvedYjs {
-  let current: unknown = rootMap
-  let schema = rootSchema
-  // Track the accumulated absolute schema path for identity lookup.
-  // Only string (key) segments contribute — index segments are structural
-  // and don't participate in identity-keying.
-  let absPath = ""
-
-  for (let i = 0; i < path.length; i++) {
-    const seg = path.segments[i]
-    if (!seg) throw new Error(`Missing segment at index ${i}`)
-    const nextSchema = advanceSchema(schema, seg)
-
-    // Compute identity for this step if binding is provided and the
-    // segment is a key (field name at a product boundary).
-    let identity: string | undefined
-    if (binding && seg.role === "key") {
-      const segStr = seg.resolve() as string
-      absPath = absPath ? `${absPath}.${segStr}` : segStr
-      identity = binding.forward.get(absPath) as string | undefined
-    }
-
-    current = stepIntoYjs(current, seg, identity)
-    schema = nextSchema
-
-    // Sum variants are always PlainSchema — no CRDT containers inside.
-    // Once we land on a sum, resolve remaining segments via plain JS
-    // property access on the (JSON) value.
-    if (schema[KIND] === "sum" && i + 1 < path.length) {
-      for (let j = i + 1; j < path.length; j++) {
-        const remaining = path.segments[j]
-        if (!remaining) throw new Error(`Missing segment at index ${j}`)
-        current = (current as Record<string, unknown>)?.[
-          remaining.resolve() as string
-        ]
-      }
-      return { resolved: current, schema }
-    }
-  }
-
-  return { resolved: current, schema }
+): PathFoldResult {
+  return foldPath(rootMap, rootSchema, path, stepIntoYjs, binding)
 }