@kyneta/yjs-schema 1.6.1 → 1.8.0

package/src/__tests__/structural-merge.test.ts

@@ -60,7 +60,7 @@ describe("structural merge protocol (Yjs)", () => {
     // Peer A creates doc and writes text
     const docA = new Y.Doc()
     docA.clientID = 100
-    ensureContainers(docA, TestSchema, false, binding)
+    ensureContainers(docA, TestSchema, binding)
     docA.transact(() => {
       const root = docA.getMap("root")
       ;(root.get(id("title")) as Y.Text).insert(0, "Hello from A")
@@ -70,7 +70,7 @@ describe("structural merge protocol (Yjs)", () => {
     // Peer B independently creates same doc and writes different text
     const docB = new Y.Doc()
     docB.clientID = 200
-    ensureContainers(docB, TestSchema, false, binding)
+    ensureContainers(docB, TestSchema, binding)
     docB.transact(() => {
       const root = docB.getMap("root")
       ;(root.get(id("title")) as Y.Text).insert(0, "Hello from B")
@@ -105,7 +105,7 @@ describe("structural merge protocol (Yjs)", () => {
     const docs = [100, 200, 300].map(cid => {
       const doc = new Y.Doc()
       doc.clientID = cid
-      ensureContainers(doc, TestSchema, false, binding)
+      ensureContainers(doc, TestSchema, binding)
       doc.transact(() => {
         const root = doc.getMap("root")
         ;(root.get(id("title")) as Y.Text).insert(0, `Peer${cid}`)
@@ -145,7 +145,7 @@ describe("structural merge protocol (Yjs)", () => {
     // Create and write
     const doc1 = new Y.Doc()
     doc1.clientID = 42
-    ensureContainers(doc1, TestSchema, false, binding)
+    ensureContainers(doc1, TestSchema, binding)
     doc1.transact(() => {
       const root = doc1.getMap("root")
       ;(root.get(id("title")) as Y.Text).insert(0, "Persistent")
@@ -159,7 +159,7 @@ describe("structural merge protocol (Yjs)", () => {
     const doc2 = new Y.Doc()
     doc2.clientID = 42
     Y.applyUpdate(doc2, snapshot)
-    ensureContainers(doc2, TestSchema, true, binding) // conditional
+    ensureContainers(doc2, TestSchema, binding)
 
     // Data preserved
     const root2 = doc2.getMap("root")
@@ -190,10 +190,10 @@ describe("structural merge protocol (Yjs)", () => {
     const bindingB = trivialBinding(schemaB)
 
     const docA = new Y.Doc()
-    ensureContainers(docA, schemaA, false, bindingA)
+    ensureContainers(docA, schemaA, bindingA)
 
     const docB = new Y.Doc()
-    ensureContainers(docB, schemaB, false, bindingB)
+    ensureContainers(docB, schemaB, bindingB)
 
     // Both should produce byte-identical structural state
     const stateA = Y.encodeStateAsUpdate(docA)
@@ -221,7 +221,7 @@ describe("structural merge protocol (Yjs)", () => {
     // Peer A: create v1, write data, export
     const docA = new Y.Doc()
     docA.clientID = 100
-    ensureContainers(docA, v1Schema, false, v1Binding)
+    ensureContainers(docA, v1Schema, v1Binding)
     docA.transact(() => {
       const root = docA.getMap("root")
       ;(root.get(id("title")) as Y.Text).insert(0, "Title")
@@ -229,17 +229,17 @@ describe("structural merge protocol (Yjs)", () => {
     })
     const v1State = Y.encodeStateAsUpdate(docA)
 
-    // Peer B: independently create v2, hydrate v1 data, conditional containers
+    // Peer B: independently create v2, hydrate v1 data, ensure containers
     const docB = new Y.Doc()
     docB.clientID = 200
     Y.applyUpdate(docB, v1State)
-    ensureContainers(docB, v2Schema, true, v2Binding) // conditional — only creates "notes"
+    ensureContainers(docB, v2Schema, v2Binding)
 
     // Another peer C: same thing independently
     const docC = new Y.Doc()
     docC.clientID = 300
     Y.applyUpdate(docC, v1State)
-    ensureContainers(docC, v2Schema, true, v2Binding)
+    ensureContainers(docC, v2Schema, v2Binding)
 
     // B and C's structural ops for "notes" should be identical (both at clientID 0)
     const stateB = Y.encodeStateAsUpdate(docB)
@@ -270,7 +270,7 @@ describe("structural merge protocol (Yjs)", () => {
 
     const doc = new Y.Doc()
     doc.clientID = 999
-    ensureContainers(doc, TestSchema, false, binding)
+    ensureContainers(doc, TestSchema, binding)
 
     // clientID should be restored
     expect(doc.clientID).toBe(999)
@@ -285,7 +285,7 @@ describe("structural merge protocol (Yjs)", () => {
 
     const doc = new Y.Doc()
     doc.clientID = 777
-    ensureContainers(doc, TestSchema, false, binding)
+    ensureContainers(doc, TestSchema, binding)
 
     // The state vector should NOT contain the caller's clientID —
     // only STRUCTURAL_YJS_CLIENT_ID (0) should have produced ops.
@@ -399,19 +399,19 @@ describe("structural merge protocol (Yjs)", () => {
     expect(rootA.get(id("count"))).toBe(rootB.get(id("count")))
   })
 
-  // ── Conditional ensureContainers is idempotent ──
+  // ── ensureContainers is idempotent ──
 
-  it("conditional ensureContainers is idempotent on hydrated doc", () => {
+  it("ensureContainers after hydration preserves existing data", () => {
     const binding = trivialBinding(TestSchema)
 
     const doc = new Y.Doc()
     doc.clientID = 50
-    ensureContainers(doc, TestSchema, false, binding)
+    ensureContainers(doc, TestSchema, binding)
 
     const stateBefore = Y.encodeStateAsUpdate(doc)
 
-    // Conditional call should not create new ops (everything already exists)
-    ensureContainers(doc, TestSchema, true, binding)
+    // Repeated call should not create new ops (everything already exists)
+    ensureContainers(doc, TestSchema, binding)
 
     const stateAfter = Y.encodeStateAsUpdate(doc)
     expect(stateAfter).toEqual(stateBefore)

package/src/__tests__/substrate.test.ts

@@ -8,6 +8,7 @@ import {
   RawPath,
   Schema,
   subscribe,
+  unwrap,
   version,
 } from "@kyneta/schema"
 import { describe, expect, it } from "vitest"
@@ -618,7 +619,7 @@ describe("YjsSubstrate", () => {
   // Context: jj:qpultxsw.
 
   describe("re-entrant write during merge replay", () => {
-    it("subscriber's local change() inside a merge-replay batch lands in Yjs", async () => {
+    it("subscriber's local change() inside a merge-replay batch lands in Yjs", () => {
       const docA = createDoc(yjs.bind(SimpleSchema))
       const docB = createDoc(yjs.bind(SimpleSchema))
 
@@ -647,11 +648,63 @@ describe("YjsSubstrate", () => {
       const delta = exportSince(docA, v0)!
       merge(docB, delta, { origin: "sync" })
 
-      await new Promise<void>(r => queueMicrotask(r))
-
       expect(docB.title()).toBe("seedmore")
       expect(docB.count()).toBe(42)
       expect(writes).toBe(1)
     })
   })
+
+  // -------------------------------------------------------------------------
+  // Origin-free discriminator tests
+  // -------------------------------------------------------------------------
+
+  describe("origin-free discriminator", () => {
+    it("options.origin survives to transaction.origin", () => {
+      const doc = createDoc(yjs.bind(SimpleSchema))
+      const native = unwrap(doc) as Y.Doc
+
+      let capturedOrigin: string | undefined = "not-called"
+      native.on("afterTransaction", tr => {
+        capturedOrigin = tr.origin
+      })
+
+      change(doc, d => d.title.insert(0, "x"), { origin: "undo" })
+      expect(capturedOrigin).toBe("undo")
+    })
+
+    it("external wrapping kyneta is correctly classified as own", () => {
+      const doc = createDoc(yjs.bind(SimpleSchema))
+
+      let kynetaFires = 0
+      subscribe(doc, () => {
+        kynetaFires++
+      })
+
+      const native = unwrap(doc) as Y.Doc
+      native.transact(() => {
+        change(doc, d => d.title.insert(0, "x"))
+      }, "external")
+
+      // Should fire exactly once (captured via wrappedPrepare),
+      // and NOT twice (the bridge should skip the external transaction
+      // because the inner kyneta transact marked the transaction).
+      expect(kynetaFires).toBe(1)
+    })
+
+    it("external raw transact with any string origin is bridged", () => {
+      const doc = createDoc(yjs.bind(SimpleSchema))
+
+      let kynetaFires = 0
+      subscribe(doc, () => {
+        kynetaFires++
+      })
+
+      const native = unwrap(doc) as Y.Doc
+      native.transact(() => {
+        native.getMap("root").set("title", "ext")
+      }, "kyneta-prepare")
+
+      expect(kynetaFires).toBe(1)
+    })
+  })
 })

package/src/bind-yjs.ts

@@ -105,17 +105,15 @@ function createYjsFactory(
       // Set stable identity AFTER hydration — avoids Yjs clientID
       // conflict detection that would reassign to a random value.
       doc.clientID = numericClientId
-      // Conditional ensureContainers: skip fields that already exist
-      // from hydrated state (each set() is a CRDT write).
-      ensureContainers(doc, schema, true, binding)
+      ensureContainers(doc, schema, binding)
       return createYjsSubstrate(doc, schema, binding)
     },
 
     create(schema: SchemaNode): Substrate<YjsVersion> {
-      // Fresh doc — set identity immediately, unconditional containers.
+      // Fresh doc — set identity immediately.
       const doc = new Y.Doc()
       doc.clientID = numericClientId
-      ensureContainers(doc, schema, false, binding)
+      ensureContainers(doc, schema, binding)
       return createYjsSubstrate(doc, schema, binding)
     },
 

package/src/change-mapping.ts

@@ -22,6 +22,7 @@ import type {
   MapChange,
   Op,
   Path,
+  ProductSchema,
   ReplaceChange,
   RichTextChange,
   RichTextInstruction,
@@ -33,9 +34,11 @@ import type {
   TextInstruction,
 } from "@kyneta/schema"
 import {
-  advanceSchema,
   expandMapOpsToLeaves,
+  isJsonBoundary,
+  isPlainObject,
   KIND,
+  pathSchema,
   RawPath,
   richTextChange,
 } from "@kyneta/schema"
@@ -118,6 +121,17 @@ export function applyChangeToYjs(
           `Attempted TreeChange at path [${pathToString(path)}].`,
       )
 
+    case "set-op":
+      // Sets (`Schema.set`) are rejected by `yjs.bind` at compile time
+      // (`"add-wins-per-key"` is not in `YjsLaws`). Unreachable from any
+      // bound Yjs substrate today; kept against the new `SetChange`
+      // vocabulary so future law-set expansion has a clear extension point.
+      throw new Error(
+        `Yjs substrate does not support "${change.type}" changes. ` +
+          `Schema.set requires "add-wins-per-key" which is not in YjsLaws. ` +
+          `Attempted SetChange at path [${pathToString(path)}].`,
+      )
+
     default:
       throw new Error(
         `applyChangeToYjs: unsupported change type "${change.type}"`,
@@ -201,7 +215,7 @@ function applySequenceChange(
   }
 
   // Resolve the item schema for structured insert detection
-  const targetSchema = resolveSchemaAtPath(rootSchema, path)
+  const targetSchema = pathSchema(rootSchema, path)
   const itemSchema = getItemSchema(targetSchema)
 
   let cursor = 0
@@ -241,7 +255,7 @@ function applyMapChange(
   }
 
   // Resolve the schema at this path for structured value detection
-  const targetSchema = resolveSchemaAtPath(rootSchema, path)
+  const targetSchema = pathSchema(rootSchema, path)
 
   // Apply deletes first
   if (change.delete) {
@@ -259,9 +273,10 @@ function applyMapChange(
       // For map schemas (records), use the key as-is (no identity-keying).
       let mapKey = key
       if (binding && targetSchema[KIND] === "product") {
-        // Compute absolute schema path for this field.
+        // Compute absolute schema path for this field — only product-field
+        // segments contribute (entry segments are runtime keys).
         const parentAbsPath = path.segments
-          .filter(s => s.role === "key")
+          .filter(s => s.role === "field")
           .map(s => s.resolve() as string)
           .join(".")
         const absPath = parentAbsPath ? `${parentAbsPath}.${key}` : key
@@ -286,7 +301,7 @@ function applyReplaceChange(
 ): void {
   if (path.length === 0) {
     throw new Error(
-      "applyChangeToYjs: ReplaceChange at root path is not supported",
+      "Cannot replace the root document struct in a CRDT backend. The root identity is fixed. Please mutate its properties individually (e.g., `doc.myField.set(value)` instead of `doc.set({ myField: value })`).",
     )
   }
 
@@ -303,15 +318,19 @@ function applyReplaceChange(
   )
 
   const resolved = lastSeg.resolve()
-  if (parent instanceof Y.Map && lastSeg.role === "key") {
+  if (
+    parent instanceof Y.Map &&
+    (lastSeg.role === "field" || lastSeg.role === "entry")
+  ) {
     // Resolve schema for the target field for structured value detection
-    const targetSchema = resolveSchemaAtPath(rootSchema, path)
+    const targetSchema = pathSchema(rootSchema, path)
     const yjsValue = maybeCreateSharedType(change.value, targetSchema)
-    // Use identity hash for product-field boundaries.
+    // Identity-keying applies only at product-field boundaries; entry
+    // segments use the runtime key as-is.
     let mapKey = resolved as string
-    if (binding) {
+    if (binding && lastSeg.role === "field") {
       const absPath = path.segments
-        .filter(s => s.role === "key")
+        .filter(s => s.role === "field")
         .map(s => s.resolve() as string)
         .join(".")
       const identity = binding.forward.get(absPath) as string | undefined
@@ -319,7 +338,7 @@ function applyReplaceChange(
     }
     parent.set(mapKey, yjsValue)
   } else if (parent instanceof Y.Array && lastSeg.role === "index") {
-    const targetSchema = resolveSchemaAtPath(rootSchema, path)
+    const targetSchema = pathSchema(rootSchema, path)
     const yjsValue = maybeCreateSharedType(change.value, targetSchema)
     parent.delete(resolved as number, 1)
     parent.insert(resolved as number, [yjsValue])
@@ -349,6 +368,12 @@ function maybeCreateSharedType(
 ): unknown {
   if (schema === undefined) return value
 
+  // JSON-boundary schemas (struct.json/list.json/record.json) store
+  // their entire subtree as a plain JSON value in the parent Y.Map
+  // entry. Skip Y.Map/Y.Array materialisation and pass the value
+  // through unchanged — including nested objects/arrays underneath.
+  if (isJsonBoundary(schema)) return value
+
   switch (schema[KIND]) {
     // First-class text → Y.Text
     case "text": {
@@ -383,12 +408,7 @@ function maybeCreateSharedType(
     }
 
     case "product": {
-      if (
-        value === null ||
-        value === undefined ||
-        typeof value !== "object" ||
-        Array.isArray(value)
-      ) {
+      if (!isPlainObject(value)) {
         return value
       }
       return createStructuredMap(value as Record<string, unknown>, schema)
@@ -406,12 +426,7 @@ function maybeCreateSharedType(
     }
 
     case "map": {
-      if (
-        value === null ||
-        value === undefined ||
-        typeof value !== "object" ||
-        Array.isArray(value)
-      ) {
+      if (!isPlainObject(value)) {
         return value
       }
       const map = new Y.Map()
@@ -509,7 +524,7 @@ export function eventsToOps(
   const ops: Op[] = []
 
   for (const event of events) {
-    const kynetaPath = yjsPathToKynetaPath(event.path, binding)
+    const kynetaPath = yjsPathToKynetaPath(event.path, schema, binding)
     const change = eventToChange(event, schema, kynetaPath, binding)
     if (change) {
       ops.push({ path: kynetaPath, change })
@@ -524,31 +539,55 @@ export function eventsToOps(
 // ---------------------------------------------------------------------------
 
 /**
- * Convert a Yjs event path (array of string | number) to a kyneta Path.
+ * Convert a Yjs event path to a kyneta `RawPath`, walking the schema
+ * alongside so each segment is classified as field / entry / index by
+ * the current schema kind.
  *
- * `event.path` from `observeDeep` is relative to the observed type.
- * Strings become key segments, numbers become index segments.
+ * Why schema-aware and not "did the inverse lookup hit?": the binding's
+ * inverse map only covers declared product-field positions reachable
+ * without crossing a runtime-keyed container. A declared struct field
+ * nested under a `record(...)` value type is reachable via Yjs but
+ * absent from `binding.inverse` — without the schema walk it would be
+ * misclassified as an entry and then rejected by `advanceSchema`.
  */
 function yjsPathToKynetaPath(
   yjsPath: (string | number)[],
+  rootSchema: SchemaNode,
   binding?: SchemaBinding,
 ): RawPath {
   let path = RawPath.empty
+  let schema: SchemaNode | undefined = rootSchema
   for (const segment of yjsPath) {
     if (typeof segment === "string") {
-      // Reverse-map identity hash → absolute schema path → leaf field name.
-      // Yjs events emit identity-keyed strings at product-field positions;
-      // we need to recover the original field name for kyneta schema paths.
+      // Inverse-lookup recovers the original declared field name when
+      // the segment IS an identity hash; otherwise we keep the string.
+      let leaf = segment
       const absPath = binding?.inverse.get(segment as any)
       if (absPath) {
         const lastDot = absPath.lastIndexOf(".")
-        const leaf = lastDot >= 0 ? absPath.slice(lastDot + 1) : absPath
+        leaf = lastDot >= 0 ? absPath.slice(lastDot + 1) : absPath
+      }
+      const kind = schema?.[KIND]
+      if (kind === "product") {
         path = path.field(leaf)
+        schema = (schema as ProductSchema | undefined)?.fields[leaf]
+      } else if (kind === "map" || kind === "set" || kind === "tree") {
+        path = path.entry(leaf)
+        schema = (schema as any)?.item
       } else {
-        path = path.field(segment)
+        // Unknown / sum / unrecognized — fall back to entry. Subsequent
+        // segments are likely walking plain JSON inside a sum variant.
+        path = path.entry(leaf)
+        schema = undefined
       }
     } else if (typeof segment === "number") {
       path = path.item(segment)
+      const kind = schema?.[KIND]
+      if (kind === "sequence" || kind === "movable") {
+        schema = (schema as any).item
+      } else {
+        schema = undefined
+      }
     }
   }
   return path
@@ -576,7 +615,7 @@ function eventToChange(
 ): ChangeBase | null {
   if (event.target instanceof Y.Text) {
     // Both text and richtext use Y.Text — resolve the schema to dispatch.
-    const schemaAtPath = resolveSchemaAtPath(rootSchema, kynetaPath)
+    const schemaAtPath = pathSchema(rootSchema, kynetaPath)
     if (schemaAtPath[KIND] === "richtext") {
       return richTextEventToChange(event)
     }
@@ -739,20 +778,6 @@ function extractEventValue(value: unknown): unknown {
 // Schema helpers
 // ---------------------------------------------------------------------------
 
-/**
- * Resolve the schema at a given path by walking through advanceSchema.
- */
-function resolveSchemaAtPath(rootSchema: SchemaNode, path: Path): SchemaNode {
-  let schema = rootSchema
-  for (const seg of path.segments) {
-    schema = advanceSchema(schema, seg)
-    // Sum variants are always PlainSchema — cannot advance further.
-    // Return early; remaining segments address plain JSON values.
-    if (schema[KIND] === "sum") return schema
-  }
-  return schema
-}
-
 /**
  * Get the item schema from a sequence schema, if available.
  */

package/src/index.ts

@@ -50,8 +50,7 @@ export type { YjsNativeMap } from "./native-map.js"
 export { ensureContainers } from "./populate.js"
 // Position conformance
 export { fromYjsAssoc, toYjsAssoc, YjsPosition } from "./position.js"
-// Reader
-export { yjsReader } from "./reader.js"
+
 // Substrate
 export {
   createYjsSubstrate,

package/src/materialize.ts

@@ -0,0 +1,109 @@
+// materialize — Yjs→PlainState materialization via generic resolver.
+//
+// Implements `createYjsResolver`, a closure-based `MaterializeResolver`
+// that navigates the Yjs shared type tree via `resolveYjsType`. The
+// generic `createMaterializeInterpreter` drives the catamorphism; the
+// resolver handles only the CRDT-specific value extraction.
+//
+// Unsupported types (counter, tree, movable) return `undefined` from
+// the resolver, triggering the generic interpreter's zero fallback.
+//
+// Zero fallback for missing values is handled canonically by the
+// generic interpreter — not inlined here.
+
+import type {
+  MaterializeResolver,
+  Path,
+  PlainState,
+  RichTextDelta,
+  SchemaBinding,
+  Schema as SchemaNode,
+} from "@kyneta/schema"
+import {
+  createMaterializeInterpreter,
+  interpret,
+  isNonNullObject,
+  materializeContextFromResolver,
+} from "@kyneta/schema"
+import * as Y from "yjs"
+import { extractValue, yTextToRichTextDelta } from "./yjs-extract.js"
+import { resolveYjsType } from "./yjs-resolve.js"
+
+// ---------------------------------------------------------------------------
+// Yjs resolver
+// ---------------------------------------------------------------------------
+
+function createYjsResolver(
+  rootMap: Y.Map<any>,
+  rootSchema: SchemaNode,
+  binding?: SchemaBinding,
+): MaterializeResolver {
+  return {
+    resolveValue(path: Path): unknown {
+      const result = resolveYjsType(rootMap, rootSchema, path, binding)
+      return extractValue(result.resolved)
+    },
+
+    resolveText(path: Path): string | undefined {
+      const { resolved } = resolveYjsType(rootMap, rootSchema, path, binding)
+      if (resolved instanceof Y.Text) {
+        return resolved.toJSON()
+      }
+      const value = extractValue(resolved)
+      return typeof value === "string" ? value : undefined
+    },
+
+    // Yjs does not support counters — schemas with counter types are
+    // rejected at bind time. Return undefined to trigger zero fallback.
+    resolveCounter(_path: Path): number | undefined {
+      return undefined
+    },
+
+    resolveRichText(path: Path): RichTextDelta | undefined {
+      const { resolved } = resolveYjsType(rootMap, rootSchema, path, binding)
+      if (resolved instanceof Y.Text) {
+        return yTextToRichTextDelta(resolved)
+      }
+      return undefined
+    },
+
+    resolveLength(path: Path): number {
+      const { resolved } = resolveYjsType(rootMap, rootSchema, path, binding)
+      if (resolved instanceof Y.Array) {
+        return resolved.length
+      }
+      return Array.isArray(resolved) ? resolved.length : 0
+    },
+
+    resolveKeys(path: Path): string[] {
+      const { resolved } = resolveYjsType(rootMap, rootSchema, path, binding)
+      if (resolved instanceof Y.Map) {
+        return Array.from(resolved.keys())
+      }
+      return isNonNullObject(resolved) ? Object.keys(resolved) : []
+    },
+
+    // Yjs has no tree primitive — schemas with `Schema.tree` are rejected
+    // at bind time. Defensive [] for any caller that reaches here.
+    resolveForest(_path: Path): readonly never[] {
+      return []
+    },
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+export function materializeYjsShadow(
+  doc: Y.Doc,
+  schema: SchemaNode,
+  binding?: SchemaBinding,
+): PlainState {
+  const rootMap = doc.getMap("root")
+  const resolver = createYjsResolver(rootMap, schema, binding)
+  const interp = createMaterializeInterpreter(resolver)
+  const ctx = materializeContextFromResolver(resolver)
+  const result = interpret(schema, interp, ctx)
+  return result as PlainState
+}