@kyneta/yjs-schema 1.6.0 → 1.7.0

package/src/__tests__/materialize.test.ts

@@ -0,0 +1,227 @@
+// materialize — Tests for materializeYjsShadow.
+//
+// Validates that the Yjs→PlainState materialization produces correct
+// plain JS objects for all supported schema types: text, scalar,
+// sequence, nested struct, and empty documents.
+//
+// Yjs does not support counter, tree, or movableList — those are skipped.
+
+import type { ProductSchema, SchemaBinding } from "@kyneta/schema"
+import {
+  BACKING_DOC,
+  change,
+  createDoc,
+  deriveSchemaBinding,
+  KIND,
+  Schema,
+  type SchemaNode,
+  SUBSTRATE,
+} from "@kyneta/schema"
+import { describe, expect, it } from "vitest"
+import type * as Y from "yjs"
+import { yjs } from "../bind-yjs.js"
+import { materializeYjsShadow } from "../materialize.js"
+
+// ===========================================================================
+// Helpers
+// ===========================================================================
+
+function trivialBinding(schema: SchemaNode): SchemaBinding {
+  if (schema[KIND] === "product") {
+    return deriveSchemaBinding(schema as ProductSchema, {})
+  }
+  return { forward: new Map(), inverse: new Map() }
+}
+
+function getYDoc(docRef: unknown): Y.Doc {
+  const substrate = (docRef as any)[SUBSTRATE]
+  return (substrate as any)[BACKING_DOC] as Y.Doc
+}
+
+// ===========================================================================
+// Test schemas
+// ===========================================================================
+
+const TextSchema = Schema.struct({
+  title: Schema.text(),
+})
+
+const ScalarSchema = Schema.struct({
+  name: Schema.string(),
+  age: Schema.number(),
+  active: Schema.boolean(),
+})
+
+const SequenceSchema = Schema.struct({
+  items: Schema.list(Schema.string()),
+})
+
+const NestedSchema = Schema.struct({
+  meta: Schema.struct({
+    author: Schema.string(),
+    version: Schema.number(),
+  }),
+})
+
+const FullSchema = Schema.struct({
+  title: Schema.text(),
+  theme: Schema.string(),
+  tags: Schema.list(Schema.string()),
+  settings: Schema.struct({
+    darkMode: Schema.boolean(),
+    fontSize: Schema.number(),
+  }),
+})
+
+// ===========================================================================
+// Tests
+// ===========================================================================
+
+describe("materializeYjsShadow", () => {
+  it("materializes text fields", () => {
+    const doc = createDoc(yjs.bind(TextSchema))
+    change(doc, (d: any) => {
+      d.title.insert(0, "hello")
+    })
+
+    const yDoc = getYDoc(doc)
+    const binding = trivialBinding(TextSchema)
+    const result = materializeYjsShadow(yDoc, TextSchema, binding)
+
+    expect(result).toEqual({ title: "hello" })
+  })
+
+  it("materializes scalar fields", () => {
+    const doc = createDoc(yjs.bind(ScalarSchema))
+    change(doc, (d: any) => {
+      d.name.set("Alice")
+      d.age.set(30)
+      d.active.set(true)
+    })
+
+    const yDoc = getYDoc(doc)
+    const binding = trivialBinding(ScalarSchema)
+    const result = materializeYjsShadow(yDoc, ScalarSchema, binding)
+
+    expect(result).toEqual({ name: "Alice", age: 30, active: true })
+  })
+
+  it("materializes sequence fields", () => {
+    const doc = createDoc(yjs.bind(SequenceSchema))
+    // Separate change() calls for list pushes to preserve order
+    // (Yjs reverses order within a single transaction)
+    change(doc, (d: any) => d.items.push("alpha"))
+    change(doc, (d: any) => d.items.push("beta"))
+    change(doc, (d: any) => d.items.push("gamma"))
+
+    const yDoc = getYDoc(doc)
+    const binding = trivialBinding(SequenceSchema)
+    const result = materializeYjsShadow(yDoc, SequenceSchema, binding)
+
+    expect(result).toEqual({ items: ["alpha", "beta", "gamma"] })
+  })
+
+  it("materializes nested struct fields", () => {
+    const doc = createDoc(yjs.bind(NestedSchema))
+    change(doc, (d: any) => {
+      d.meta.author.set("Bob")
+      d.meta.version.set(42)
+    })
+
+    const yDoc = getYDoc(doc)
+    const binding = trivialBinding(NestedSchema)
+    const result = materializeYjsShadow(yDoc, NestedSchema, binding)
+
+    expect(result).toEqual({
+      meta: { author: "Bob", version: 42 },
+    })
+  })
+
+  it("materializes empty document to structural zeros", () => {
+    const doc = createDoc(yjs.bind(FullSchema))
+
+    const yDoc = getYDoc(doc)
+    const binding = trivialBinding(FullSchema)
+    const result = materializeYjsShadow(yDoc, FullSchema, binding)
+
+    expect(result).toEqual({
+      title: "",
+      theme: "",
+      tags: [],
+      settings: { darkMode: false, fontSize: 0 },
+    })
+  })
+
+  it("uses raw field names, not identity hashes", () => {
+    const doc = createDoc(yjs.bind(TextSchema))
+    change(doc, (d: any) => {
+      d.title.insert(0, "test")
+    })
+
+    const yDoc = getYDoc(doc)
+    const binding = trivialBinding(TextSchema)
+    const result = materializeYjsShadow(yDoc, TextSchema, binding)
+
+    // Keys should be the raw field name "title", not an identity hash
+    const keys = Object.keys(result as Record<string, unknown>)
+    expect(keys).toContain("title")
+    expect(keys).toHaveLength(1)
+    // No key should look like a hash (long alphanumeric string)
+    for (const key of keys) {
+      expect(key).toBe("title")
+    }
+  })
+
+  it("materializes a complex document with multiple field types", () => {
+    const doc = createDoc(yjs.bind(FullSchema))
+
+    change(doc, (d: any) => {
+      d.title.insert(0, "My Doc")
+      d.theme.set("dark")
+      d.settings.darkMode.set(true)
+      d.settings.fontSize.set(16)
+    })
+    change(doc, (d: any) => d.tags.push("important"))
+    change(doc, (d: any) => d.tags.push("urgent"))
+
+    const yDoc = getYDoc(doc)
+    const binding = trivialBinding(FullSchema)
+    const result = materializeYjsShadow(yDoc, FullSchema, binding)
+
+    expect(result).toEqual({
+      title: "My Doc",
+      theme: "dark",
+      tags: ["important", "urgent"],
+      settings: { darkMode: true, fontSize: 16 },
+    })
+  })
+
+  it("materializes without binding (undefined binding)", () => {
+    const doc = createDoc(yjs.bind(TextSchema))
+    change(doc, (d: any) => {
+      d.title.insert(0, "no binding")
+    })
+
+    const yDoc = getYDoc(doc)
+    // Pass no binding — materialize should still work
+    const result = materializeYjsShadow(yDoc, TextSchema)
+
+    // Without binding, keys may be identity hashes — but the values
+    // should still be correct and the result should be an object.
+    expect(result).toBeDefined()
+    expect(typeof result).toBe("object")
+  })
+
+  it("nested nullable materializes to null on fresh doc", () => {
+    const schema = Schema.struct({
+      settings: Schema.struct({
+        theme: Schema.string().nullable(),
+      }),
+    })
+    const doc = createDoc(yjs.bind(schema))
+    const yDoc = getYDoc(doc)
+    const binding = trivialBinding(schema)
+    const result = materializeYjsShadow(yDoc, schema, binding)
+    expect(result).toEqual({ settings: { theme: null } })
+  })
+})

package/src/__tests__/position.test.ts

@@ -20,7 +20,7 @@ import {
 import {
   type PositionTestEnv,
   positionConformance,
-} from "@kyneta/schema/src/__tests__/position-conformance.js"
+} from "@kyneta/schema/testing"
 import { describe, expect, it } from "vitest"
 import * as Y from "yjs"
 import { ensureContainers } from "../populate.js"
@@ -118,7 +118,7 @@ describe("YjsPosition: concurrent edits", () => {
     // --- Doc 2: fork from doc1's state ---
     const doc2 = new Y.Doc()
     Y.applyUpdate(doc2, Y.encodeStateAsUpdate(doc1))
-    ensureContainers(doc2, TextSchema, true)
+    ensureContainers(doc2, TextSchema)
     const substrate2 = createYjsSubstrate(doc2, TextSchema)
     const ref2 = createRef(TextSchema, substrate2) as any
 
@@ -175,7 +175,7 @@ describe("YjsPosition: concurrent edits", () => {
     // --- Doc 2: fork ---
     const doc2 = new Y.Doc()
     Y.applyUpdate(doc2, Y.encodeStateAsUpdate(doc1))
-    ensureContainers(doc2, TextSchema, true)
+    ensureContainers(doc2, TextSchema)
     const substrate2 = createYjsSubstrate(doc2, TextSchema)
     const ref2 = createRef(TextSchema, substrate2) as any
 
@@ -229,7 +229,7 @@ describe("YjsPosition: concurrent edits", () => {
     // --- Doc 2: fork ---
     const doc2 = new Y.Doc()
     Y.applyUpdate(doc2, Y.encodeStateAsUpdate(doc1))
-    ensureContainers(doc2, TextSchema, true)
+    ensureContainers(doc2, TextSchema)
     const substrate2 = createYjsSubstrate(doc2, TextSchema)
     const ref2 = createRef(TextSchema, substrate2) as any
 
@@ -285,7 +285,7 @@ describe("YjsPosition: concurrent edits", () => {
     // Decode on a different doc that has the same state
     const doc2 = new Y.Doc()
     Y.applyUpdate(doc2, Y.encodeStateAsUpdate(doc1))
-    ensureContainers(doc2, TextSchema, true)
+    ensureContainers(doc2, TextSchema)
     const substrate2 = createYjsSubstrate(doc2, TextSchema)
     const ref2 = createRef(TextSchema, substrate2) as any
 
@@ -310,13 +310,13 @@ describe("YjsPosition: concurrent edits", () => {
 
     const doc2 = new Y.Doc()
     Y.applyUpdate(doc2, Y.encodeStateAsUpdate(doc1))
-    ensureContainers(doc2, TextSchema, true)
+    ensureContainers(doc2, TextSchema)
     const substrate2 = createYjsSubstrate(doc2, TextSchema)
     const ref2 = createRef(TextSchema, substrate2) as any
 
     const doc3 = new Y.Doc()
     Y.applyUpdate(doc3, Y.encodeStateAsUpdate(doc1))
-    ensureContainers(doc3, TextSchema, true)
+    ensureContainers(doc3, TextSchema)
     const substrate3 = createYjsSubstrate(doc3, TextSchema)
     const ref3 = createRef(TextSchema, substrate3) as any
 

package/src/__tests__/record-text-spike.test.ts

@@ -147,7 +147,7 @@ describe("record-of-struct (plain baseline)", () => {
 
     const delta = exportSince(docA, v0)
     expect(delta).not.toBeNull()
-    merge(docB, delta!, "sync")
+    merge(docB, delta!, { origin: "sync" })
 
     expect(docB.profiles()).toEqual({
       alice: { displayName: "Alice", age: 30 },
@@ -304,7 +304,7 @@ describe("text-inside-struct-inside-record", () => {
 
     const delta = exportSince(docA, v0)
     expect(delta).not.toBeNull()
-    merge(docB, delta!, "sync")
+    merge(docB, delta!, { origin: "sync" })
 
     expect(docB.profiles()).toEqual({
       alice: { displayName: "Alice", bio: "Hello from A" },
@@ -342,8 +342,8 @@ describe("text-inside-struct-inside-record", () => {
     const deltaBA = exportSince(docB, vA)
     expect(deltaAB).not.toBeNull()
     expect(deltaBA).not.toBeNull()
-    merge(docB, deltaAB!, "sync")
-    merge(docA, deltaBA!, "sync")
+    merge(docB, deltaAB!, { origin: "sync" })
+    merge(docA, deltaBA!, { origin: "sync" })
 
     // Both converge to the same value (order depends on client IDs)
     expect((docA as any).profiles.at("alice").bio()).toBe(
@@ -429,7 +429,7 @@ describe("text-inside-struct-inside-list", () => {
 
     const delta = exportSince(docA, v0)
     expect(delta).not.toBeNull()
-    merge(docB, delta!, "sync")
+    merge(docB, delta!, { origin: "sync" })
 
     expect(docB.players.length).toBe(1)
     expect((docB as any).players.at(0).name()).toBe("Alice")

package/src/__tests__/structural-merge.test.ts

@@ -60,7 +60,7 @@ describe("structural merge protocol (Yjs)", () => {
     // Peer A creates doc and writes text
     const docA = new Y.Doc()
     docA.clientID = 100
-    ensureContainers(docA, TestSchema, false, binding)
+    ensureContainers(docA, TestSchema, binding)
     docA.transact(() => {
       const root = docA.getMap("root")
       ;(root.get(id("title")) as Y.Text).insert(0, "Hello from A")
@@ -70,7 +70,7 @@ describe("structural merge protocol (Yjs)", () => {
     // Peer B independently creates same doc and writes different text
     const docB = new Y.Doc()
     docB.clientID = 200
-    ensureContainers(docB, TestSchema, false, binding)
+    ensureContainers(docB, TestSchema, binding)
     docB.transact(() => {
       const root = docB.getMap("root")
       ;(root.get(id("title")) as Y.Text).insert(0, "Hello from B")
@@ -105,7 +105,7 @@ describe("structural merge protocol (Yjs)", () => {
     const docs = [100, 200, 300].map(cid => {
       const doc = new Y.Doc()
       doc.clientID = cid
-      ensureContainers(doc, TestSchema, false, binding)
+      ensureContainers(doc, TestSchema, binding)
       doc.transact(() => {
         const root = doc.getMap("root")
         ;(root.get(id("title")) as Y.Text).insert(0, `Peer${cid}`)
@@ -145,7 +145,7 @@ describe("structural merge protocol (Yjs)", () => {
     // Create and write
     const doc1 = new Y.Doc()
     doc1.clientID = 42
-    ensureContainers(doc1, TestSchema, false, binding)
+    ensureContainers(doc1, TestSchema, binding)
     doc1.transact(() => {
       const root = doc1.getMap("root")
       ;(root.get(id("title")) as Y.Text).insert(0, "Persistent")
@@ -159,7 +159,7 @@ describe("structural merge protocol (Yjs)", () => {
     const doc2 = new Y.Doc()
     doc2.clientID = 42
     Y.applyUpdate(doc2, snapshot)
-    ensureContainers(doc2, TestSchema, true, binding) // conditional
+    ensureContainers(doc2, TestSchema, binding)
 
     // Data preserved
     const root2 = doc2.getMap("root")
@@ -190,10 +190,10 @@ describe("structural merge protocol (Yjs)", () => {
     const bindingB = trivialBinding(schemaB)
 
     const docA = new Y.Doc()
-    ensureContainers(docA, schemaA, false, bindingA)
+    ensureContainers(docA, schemaA, bindingA)
 
     const docB = new Y.Doc()
-    ensureContainers(docB, schemaB, false, bindingB)
+    ensureContainers(docB, schemaB, bindingB)
 
     // Both should produce byte-identical structural state
     const stateA = Y.encodeStateAsUpdate(docA)
@@ -221,7 +221,7 @@ describe("structural merge protocol (Yjs)", () => {
     // Peer A: create v1, write data, export
     const docA = new Y.Doc()
     docA.clientID = 100
-    ensureContainers(docA, v1Schema, false, v1Binding)
+    ensureContainers(docA, v1Schema, v1Binding)
     docA.transact(() => {
       const root = docA.getMap("root")
       ;(root.get(id("title")) as Y.Text).insert(0, "Title")
@@ -229,17 +229,17 @@ describe("structural merge protocol (Yjs)", () => {
     })
     const v1State = Y.encodeStateAsUpdate(docA)
 
-    // Peer B: independently create v2, hydrate v1 data, conditional containers
+    // Peer B: independently create v2, hydrate v1 data, ensure containers
     const docB = new Y.Doc()
     docB.clientID = 200
     Y.applyUpdate(docB, v1State)
-    ensureContainers(docB, v2Schema, true, v2Binding) // conditional — only creates "notes"
+    ensureContainers(docB, v2Schema, v2Binding)
 
     // Another peer C: same thing independently
     const docC = new Y.Doc()
     docC.clientID = 300
     Y.applyUpdate(docC, v1State)
-    ensureContainers(docC, v2Schema, true, v2Binding)
+    ensureContainers(docC, v2Schema, v2Binding)
 
     // B and C's structural ops for "notes" should be identical (both at clientID 0)
     const stateB = Y.encodeStateAsUpdate(docB)
@@ -270,7 +270,7 @@ describe("structural merge protocol (Yjs)", () => {
 
     const doc = new Y.Doc()
     doc.clientID = 999
-    ensureContainers(doc, TestSchema, false, binding)
+    ensureContainers(doc, TestSchema, binding)
 
     // clientID should be restored
     expect(doc.clientID).toBe(999)
@@ -285,7 +285,7 @@ describe("structural merge protocol (Yjs)", () => {
 
     const doc = new Y.Doc()
     doc.clientID = 777
-    ensureContainers(doc, TestSchema, false, binding)
+    ensureContainers(doc, TestSchema, binding)
 
     // The state vector should NOT contain the caller's clientID —
     // only STRUCTURAL_YJS_CLIENT_ID (0) should have produced ops.
@@ -383,8 +383,8 @@ describe("structural merge protocol (Yjs)", () => {
     // Bidirectional merge — should not throw
     const payloadA = subA.exportEntirety()
     const payloadB = subB.exportEntirety()
-    subA.merge(payloadB, "sync")
-    subB.merge(payloadA, "sync")
+    subA.merge(payloadB, { origin: "sync" })
+    subB.merge(payloadA, { origin: "sync" })
 
     // Both converge
     const rootA = docA.getMap("root")
@@ -399,19 +399,19 @@ describe("structural merge protocol (Yjs)", () => {
     expect(rootA.get(id("count"))).toBe(rootB.get(id("count")))
   })
 
-  // ── Conditional ensureContainers is idempotent ──
+  // ── ensureContainers is idempotent ──
 
-  it("conditional ensureContainers is idempotent on hydrated doc", () => {
+  it("ensureContainers after hydration preserves existing data", () => {
     const binding = trivialBinding(TestSchema)
 
     const doc = new Y.Doc()
     doc.clientID = 50
-    ensureContainers(doc, TestSchema, false, binding)
+    ensureContainers(doc, TestSchema, binding)
 
     const stateBefore = Y.encodeStateAsUpdate(doc)
 
-    // Conditional call should not create new ops (everything already exists)
-    ensureContainers(doc, TestSchema, true, binding)
+    // Repeated call should not create new ops (everything already exists)
+    ensureContainers(doc, TestSchema, binding)
 
     const stateAfter = Y.encodeStateAsUpdate(doc)
     expect(stateAfter).toEqual(stateBefore)

package/src/__tests__/substrate.test.ts

@@ -607,4 +607,49 @@ describe("YjsSubstrate", () => {
       expect(parsed.compare(v)).toBe("equal")
     })
   })
+
+  // -------------------------------------------------------------------------
+  // Re-entrant write during merge replay
+  // -------------------------------------------------------------------------
+  //
+  // A subscriber that calls `change(doc, ...)` while delivering a sync
+  // merge must reach Yjs — otherwise the substrate stalls and the
+  // subscriber loops on stale state until the lease budget trips.
+  // Context: jj:qpultxsw.
+
+  describe("re-entrant write during merge replay", () => {
+    it("subscriber's local change() inside a merge-replay batch lands in Yjs", () => {
+      const docA = createDoc(yjs.bind(SimpleSchema))
+      const docB = createDoc(yjs.bind(SimpleSchema))
+
+      change(docA, (d: any) => {
+        d.title.insert(0, "seed")
+      })
+      merge(docB, exportEntirety(docA), { origin: "sync" })
+
+      // On the first replay-driven update, the subscriber writes once
+      // to an unrelated field. The write must hit Yjs; the guard
+      // ensures we don't re-enter on subsequent flushes.
+      let writes = 0
+      subscribe(docB.title, () => {
+        if (writes === 0 && (docB.title() as string) === "seedmore") {
+          writes++
+          change(docB, (d: any) => {
+            d.count.set(42)
+          })
+        }
+      })
+
+      const v0 = version(docB)
+      change(docA, (d: any) => {
+        d.title.insert((d.title() as string).length, "more")
+      })
+      const delta = exportSince(docA, v0)!
+      merge(docB, delta, { origin: "sync" })
+
+      expect(docB.title()).toBe("seedmore")
+      expect(docB.count()).toBe(42)
+      expect(writes).toBe(1)
+    })
+  })
 })

package/src/bind-yjs.ts

@@ -105,17 +105,15 @@ function createYjsFactory(
       // Set stable identity AFTER hydration — avoids Yjs clientID
       // conflict detection that would reassign to a random value.
       doc.clientID = numericClientId
-      // Conditional ensureContainers: skip fields that already exist
-      // from hydrated state (each set() is a CRDT write).
-      ensureContainers(doc, schema, true, binding)
+      ensureContainers(doc, schema, binding)
       return createYjsSubstrate(doc, schema, binding)
     },
 
     create(schema: SchemaNode): Substrate<YjsVersion> {
-      // Fresh doc — set identity immediately, unconditional containers.
+      // Fresh doc — set identity immediately.
       const doc = new Y.Doc()
       doc.clientID = numericClientId
-      ensureContainers(doc, schema, false, binding)
+      ensureContainers(doc, schema, binding)
       return createYjsSubstrate(doc, schema, binding)
     },