@kyneta/yjs-schema 1.3.0 → 1.4.0

package/src/reader.ts

@@ -7,8 +7,23 @@
 //
 // Y.Text → .toJSON() (string), Y.Map → .toJSON() (plain object),
 // Y.Array → .toJSON() (plain array), plain values → as-is.
+//
+// Richtext: when the schema at the resolved path is "richtext" and
+// the resolved value is a Y.Text, we call .toDelta() and convert to
+// RichTextDelta (array of { text, marks? } spans) instead of .toJSON().
+//
+// Identity-keying: when a SchemaBinding is provided, resolveYjsType
+// navigates Y.Map children using identity hashes instead of field names.
 
-import type { Path, Reader, Schema as SchemaNode } from "@kyneta/schema"
+import type {
+  Path,
+  Reader,
+  RichTextDelta,
+  RichTextSpan,
+  SchemaBinding,
+  Schema as SchemaNode,
+} from "@kyneta/schema"
+import { KIND } from "@kyneta/schema"
 import * as Y from "yjs"
 import { resolveYjsType } from "./yjs-resolve.js"
 
@@ -38,6 +53,33 @@ function extractValue(resolved: unknown): unknown {
   return resolved
 }
 
+// ---------------------------------------------------------------------------
+// Rich text delta conversion
+// ---------------------------------------------------------------------------
+
+/**
+ * Convert a Y.Text's delta (Quill format) to a kyneta RichTextDelta.
+ *
+ * Yjs `.toDelta()` returns `{ insert: string, attributes?: Record<string, any> }[]`.
+ * Kyneta RichTextDelta is `{ text: string, marks?: MarkMap }[]`.
+ */
+function yTextToRichTextDelta(ytext: Y.Text): RichTextDelta {
+  const delta = ytext.toDelta() as Array<{
+    insert: string
+    attributes?: Record<string, unknown>
+  }>
+  const spans: RichTextSpan[] = []
+  for (const d of delta) {
+    if (typeof d.insert !== "string") continue
+    const span: RichTextSpan =
+      d.attributes && Object.keys(d.attributes).length > 0
+        ? { text: d.insert, marks: d.attributes }
+        : { text: d.insert }
+    spans.push(span)
+  }
+  return spans
+}
+
 // ---------------------------------------------------------------------------
 // yjsReader
 // ---------------------------------------------------------------------------
@@ -55,8 +97,13 @@ function extractValue(resolved: unknown): unknown {
  *
  * @param doc - The Y.Doc to read from.
  * @param schema - The root schema for the document.
+ * @param binding - Optional SchemaBinding for identity-keyed navigation.
  */
-export function yjsReader(doc: Y.Doc, schema: SchemaNode): Reader {
+export function yjsReader(
+  doc: Y.Doc,
+  schema: SchemaNode,
+  binding?: SchemaBinding,
+): Reader {
   const rootMap = doc.getMap("root")
 
   return {
@@ -65,12 +112,21 @@ export function yjsReader(doc: Y.Doc, schema: SchemaNode): Reader {
         // Root read — return the full root map as JSON
         return rootMap.toJSON()
       }
-      const resolved = resolveYjsType(rootMap, schema, path)
+      const { resolved, schema: nodeSchema } = resolveYjsType(
+        rootMap,
+        schema,
+        path,
+        binding,
+      )
+      // Richtext: Y.Text at a richtext schema position → RichTextDelta
+      if (nodeSchema[KIND] === "richtext" && resolved instanceof Y.Text) {
+        return yTextToRichTextDelta(resolved)
+      }
       return extractValue(resolved)
     },
 
     arrayLength(path: Path): number {
-      const resolved = resolveYjsType(rootMap, schema, path)
+      const { resolved } = resolveYjsType(rootMap, schema, path, binding)
       if (resolved instanceof Y.Array) {
         return resolved.length
       }
@@ -82,7 +138,7 @@ export function yjsReader(doc: Y.Doc, schema: SchemaNode): Reader {
     },
 
     keys(path: Path): string[] {
-      const resolved = resolveYjsType(rootMap, schema, path)
+      const { resolved } = resolveYjsType(rootMap, schema, path, binding)
       if (resolved instanceof Y.Map) {
         return Array.from(resolved.keys())
       }
@@ -99,7 +155,7 @@ export function yjsReader(doc: Y.Doc, schema: SchemaNode): Reader {
     },
 
     hasKey(path: Path, key: string): boolean {
-      const resolved = resolveYjsType(rootMap, schema, path)
+      const { resolved } = resolveYjsType(rootMap, schema, path, binding)
       if (resolved instanceof Y.Map) {
         return resolved.has(key)
       }

package/src/substrate.ts

@@ -9,14 +9,22 @@
 // means subscribing to the kyneta doc observes ALL mutations to the
 // underlying Y.Doc, regardless of source (local kyneta writes,
 // merge, external Y.applyUpdate, external raw Yjs API mutations).
+//
+// Identity-keying: when a SchemaBinding is provided, all Y.Map key
+// lookups and writes use the identity hash instead of the field name.
+// The binding is threaded to the reader, event bridge, and write path.
 
 import type {
   ChangeBase,
   Path,
+  PositionCapable,
+  ProductSchema,
   Reader,
   Replica,
   ReplicaFactory,
+  SchemaBinding,
   Schema as SchemaNode,
+  Side,
   Substrate,
   SubstrateFactory,
   SubstratePayload,
@@ -25,12 +33,14 @@ import type {
 import {
   BACKING_DOC,
   buildWritableContext,
+  deriveSchemaBinding,
   executeBatch,
   KIND,
 } from "@kyneta/schema"
 import * as Y from "yjs"
 import { applyChangeToYjs, eventsToOps } from "./change-mapping.js"
 import { ensureContainers } from "./populate.js"
+import { toYjsAssoc, YjsPosition } from "./position.js"
 import { yjsReader } from "./reader.js"
 import { YjsVersion } from "./version.js"
 import { resolveYjsType } from "./yjs-resolve.js"
@@ -62,10 +72,12 @@ const KYNETA_ORIGIN = "kyneta-prepare"
  * @param doc - The Y.Doc to wrap. The substrate does NOT own the doc;
  *   the caller is responsible for its lifecycle.
  * @param schema - The root schema for the document.
+ * @param binding - Optional SchemaBinding for identity-keyed containers.
  */
 export function createYjsSubstrate(
   doc: Y.Doc,
   schema: SchemaNode,
+  binding?: SchemaBinding,
 ): Substrate<YjsVersion> {
   // --- Closure-scoped state ---
 
@@ -84,11 +96,19 @@ export function createYjsSubstrate(
   // Lazy-built WritableContext (same pattern as PlainSubstrate / LoroSubstrate).
   let cachedCtx: WritableContext | undefined
 
+  // Incremental delete set tracking.
+  // Initialized once from the struct store (O(n)), then maintained
+  // incrementally by merging each transaction's deleteSet (O(delta)).
+  // Note: DeleteSet class isn't exported from yjs's public entry point,
+  // so we infer the type from createDeleteSet's return type.
+  let accumulatedDs: ReturnType<typeof Y.createDeleteSet> =
+    Y.createDeleteSetFromStructStore(doc.store)
+
   // The root Y.Map — all schema fields are children of this single map.
   const rootMap = doc.getMap("root")
 
   // The Reader — live view over the Yjs shared type tree.
-  const reader: Reader = yjsReader(doc, schema)
+  const reader: Reader = yjsReader(doc, schema, binding)
 
   // --- Substrate object ---
 
@@ -115,7 +135,7 @@ export function createYjsSubstrate(
         try {
           doc.transact(() => {
             for (const { path, change } of pendingChanges) {
-              applyChangeToYjs(rootMap, schema, path, change)
+              applyChangeToYjs(rootMap, schema, path, change, binding)
             }
           }, KYNETA_ORIGIN)
           pendingChanges.length = 0
@@ -139,14 +159,46 @@ export function createYjsSubstrate(
           if (path.segments.length === 0) return doc
           if (nodeSchema[KIND] === "scalar" || nodeSchema[KIND] === "sum")
             return undefined
-          return resolveYjsType(rootMap, schema, path as any)
+          return resolveYjsType(rootMap, schema, path as any, binding).resolved
+        }
+        ;(cachedCtx as any).positionResolver = (
+          _nodeSchema: unknown,
+          path: { segments: readonly unknown[] },
+        ) => {
+          return {
+            createPosition(index: number, side: Side) {
+              // Resolve path to the Y.Text shared type
+              const { resolved: ytype } = resolveYjsType(
+                rootMap,
+                schema,
+                path as any,
+                binding,
+              )
+              if (!(ytype instanceof Y.Text)) {
+                throw new Error(
+                  `positionResolver: path does not resolve to a Y.Text`,
+                )
+              }
+              const assoc = toYjsAssoc(side)
+              const rpos = Y.createRelativePositionFromTypeIndex(
+                ytype,
+                index,
+                assoc,
+              )
+              return new YjsPosition(rpos, doc)
+            },
+            decodePosition(bytes: Uint8Array) {
+              const rpos = Y.decodeRelativePosition(bytes)
+              return new YjsPosition(rpos, doc)
+            },
+          } satisfies PositionCapable
         }
       }
       return cachedCtx
     },
 
     version(): YjsVersion {
-      return new YjsVersion(Y.encodeStateVector(doc))
+      return YjsVersion.fromDeleteSet(doc, accumulatedDs)
     },
 
     baseVersion(): YjsVersion {
@@ -209,11 +261,17 @@ export function createYjsSubstrate(
     }
 
     // Convert Yjs events → kyneta Ops
-    const ops = eventsToOps(events, schema)
+    const ops = eventsToOps(events, schema, binding)
     if (ops.length === 0) {
       return
     }
 
+    // Update accumulated delete set BEFORE executeBatch, so version()
+    // reflects deletes when notifyLocalChange fires from the changefeed.
+    if (transaction.deleteSet.clients.size > 0) {
+      accumulatedDs = Y.mergeDeleteSets([accumulatedDs, transaction.deleteSet])
+    }
+
     // Determine origin: prefer stashed kyneta origin (from merge),
     // fall back to the transaction's origin if it's a string.
     const origin =
@@ -234,6 +292,20 @@ export function createYjsSubstrate(
     }
   })
 
+  // For local mutations (KYNETA_ORIGIN): the observeDeep handler returns
+  // early, so we merge the delete set via afterTransaction instead.
+  // afterTransaction fires inside doc.transact() — before onFlush returns
+  // — so accumulatedDs is up to date when the changefeed's
+  // deliverNotifications → notifyLocalChange → version() fires.
+  doc.on("afterTransaction", (transaction: Y.Transaction) => {
+    if (
+      transaction.origin === KYNETA_ORIGIN &&
+      transaction.deleteSet.clients.size > 0
+    ) {
+      accumulatedDs = Y.mergeDeleteSets([accumulatedDs, transaction.deleteSet])
+    }
+  })
+
   return substrate as Substrate<YjsVersion>
 }
 
@@ -250,7 +322,21 @@ export function createYjsSubstrate(
  * - `fromEntirety(payload, schema)` — creates a Y.Doc from an entirety
  *   payload, returns a substrate.
  * - `parseVersion(serialized)` — deserializes a YjsVersion.
+ *
+ * Uses trivialBinding for identity-keying: every path maps to
+ * `deriveIdentity(path, 1)` (generation 1, no renames).
  */
+
+/**
+ * Compute a trivial SchemaBinding for a schema with no migration history.
+ * Every product field maps to `deriveIdentity(path, 1)`.
+ */
+function trivialBinding(schema: SchemaNode): SchemaBinding {
+  if (schema[KIND] === "product") {
+    return deriveSchemaBinding(schema as ProductSchema, {})
+  }
+  return { forward: new Map(), inverse: new Map() }
+}
 // ---------------------------------------------------------------------------
 // yjsReplicaFactory — ReplicaFactory<YjsVersion>
 // ---------------------------------------------------------------------------
@@ -277,7 +363,7 @@ export function createYjsReplica(doc: Y.Doc): Replica<YjsVersion> {
     },
 
     version(): YjsVersion {
-      return new YjsVersion(Y.encodeStateVector(currentDoc))
+      return YjsVersion.fromDoc(currentDoc)
     },
 
     baseVersion(): YjsVersion {
@@ -303,7 +389,7 @@ export function createYjsReplica(doc: Y.Doc): Replica<YjsVersion> {
       const newDoc = new Y.Doc()
       Y.applyUpdate(newDoc, update)
       currentDoc = newDoc
-      currentBase = new YjsVersion(Y.encodeStateVector(currentDoc))
+      currentBase = YjsVersion.fromDoc(currentDoc)
     },
 
     exportEntirety(): SubstratePayload {
@@ -381,18 +467,20 @@ export const yjsSubstrateFactory: SubstrateFactory<YjsVersion> = {
     schema: SchemaNode,
   ): Substrate<YjsVersion> {
     const doc = (replica as any)[BACKING_DOC] as Y.Doc
+    const binding = trivialBinding(schema)
     // No identity injection for the standalone factory (no peerId).
     // Conditional ensureContainers: skip fields that already exist
     // from hydrated state.
-    ensureContainers(doc, schema, true)
-    return createYjsSubstrate(doc, schema)
+    ensureContainers(doc, schema, true, binding)
+    return createYjsSubstrate(doc, schema, binding)
   },
 
   create(schema: SchemaNode): Substrate<YjsVersion> {
     // Fresh doc — unconditional ensureContainers (nothing to conflict with).
     const doc = new Y.Doc()
-    ensureContainers(doc, schema)
-    return createYjsSubstrate(doc, schema)
+    const binding = trivialBinding(schema)
+    ensureContainers(doc, schema, false, binding)
+    return createYjsSubstrate(doc, schema, binding)
   },
 
   fromEntirety(

package/src/version.ts

@@ -1,15 +1,20 @@
-// YjsVersion — Version implementation wrapping Yjs state vectors.
+// YjsVersion — Version wrapping a Yjs snapshot (state vector + delete set).
 //
-// Yjs state vectors (`Y.encodeStateVector(doc)`) are the complete peer
-// state used for sync diffing — matching the semantics of kyneta's
-// Version interface.
+// The Yjs state vector only tracks inserted items — it does NOT advance
+// when items are deleted (tombstoned). A version based on the state vector
+// alone cannot detect delete-only changes, causing the sync protocol to
+// skip pushing deletes to peers.
 //
-// Serialization uses base64-encoded bytes for text-safe embedding in
-// HTML meta tags, script tags, etc.
+// YjsVersion wraps the full Yjs Snapshot (state vector + delete set) so
+// that compare() faithfully distinguishes "same state" from "divergent
+// deletes." The state vector component is kept separately for exportSince(),
+// which uses it to compute the minimal update payload.
 //
-// Yjs does not export a state vector comparison function, so we
-// implement standard version-vector partial-order comparison over
-// decoded `Map<number, number>` (clientID → clock) maps ourselves.
+// Serialization format: base64(sv) + "." + base64(snapshotBytes).
+// Legacy format (no "."): base64(sv) only — decoded as SV-only version
+// for backward compatibility. When a legacy version is compared against
+// a new-format version with matching SVs, the differing snapshot bytes
+// yield "concurrent", triggering a (redundant but safe) sync push.
 
 import type { Version } from "@kyneta/schema"
 import {
@@ -18,7 +23,14 @@ import {
   versionVectorCompare,
   versionVectorMeet,
 } from "@kyneta/schema"
-import { decodeStateVector } from "yjs"
+import {
+  createSnapshot,
+  type Doc,
+  decodeStateVector,
+  encodeSnapshot,
+  encodeStateVector as yjsEncodeStateVector,
+  snapshot as yjsSnapshot,
+} from "yjs"
 
 // ---------------------------------------------------------------------------
 // State vector encoding — manual varint (unsigned LEB128)
@@ -53,44 +65,112 @@ function encodeStateVector(map: Map<number, number>): Uint8Array {
   return new Uint8Array(bytes)
 }
 
+// ---------------------------------------------------------------------------
+// Byte-level equality
+// ---------------------------------------------------------------------------
+
+function arraysEqual(a: Uint8Array, b: Uint8Array): boolean {
+  if (a.length !== b.length) return false
+  for (let i = 0; i < a.length; i++) {
+    if (a[i] !== b[i]) return false
+  }
+  return true
+}
+
 // ---------------------------------------------------------------------------
 // YjsVersion
 // ---------------------------------------------------------------------------
 
 /**
- * A Version wrapping a Yjs state vector.
+ * A Version wrapping a Yjs snapshot (state vector + delete set).
+ *
+ * The state vector tracks which insertions from each client have been
+ * observed. The delete set tracks which of those items have been
+ * tombstoned. Together they fully describe a Yjs document's state.
  *
- * State vectors track the complete peer state — which operations from
- * each client have been observed. This is the right abstraction for sync
- * diffing: `exportSince(version)` uses the state vector to compute the
- * minimal update payload via `Y.encodeStateAsUpdate(doc, sv)`.
+ * - `sv` is used by `exportSince()` to compute the minimal update payload
+ *   via `Y.encodeStateAsUpdate(doc, sv)`.
+ * - `snapshotBytes` is the encoded Yjs `Snapshot` (SV + delete set),
+ *   used for equality comparison: two documents are "equal" only when
+ *   both their inserts and deletes match.
  *
- * `serialize()` encodes to base64 for text-safe embedding.
- * `compare()` decodes both state vectors and performs standard
- * version-vector partial-order comparison over the client-clock maps.
+ * `compare()` first performs standard version-vector partial-order
+ * comparison on the state vectors. If the SVs are equal, it falls
+ * through to a byte-level comparison of the snapshot bytes — if they
+ * differ (same inserts, different deletes), the result is "concurrent",
+ * ensuring the sync protocol pushes the divergent deletes.
  */
 export class YjsVersion implements Version {
+  /** Encoded state vector — used by exportSince(). */
   readonly sv: Uint8Array
 
-  constructor(sv: Uint8Array) {
+  /**
+   * Encoded Yjs snapshot (state vector + delete set) — used for equality
+   * comparison. Two documents are "equal" only if both their inserts
+   * (state vector) and deletes (delete set) match.
+   */
+  readonly snapshotBytes: Uint8Array
+
+  constructor(sv: Uint8Array, snapshotBytes?: Uint8Array) {
     this.sv = sv
+    // If no snapshot provided, use sv as snapshot (backward compat / SV-only).
+    this.snapshotBytes = snapshotBytes ?? sv
   }
 
   /**
-   * Serialize the state vector to a base64 string.
+   * Construct a version from a live `Y.Doc` by snapshotting its full state.
    *
-   * The encoding is: raw state vector bytes → base64.
-   * This is text-safe for embedding in HTML meta tags, URL parameters, etc.
+   * Walks the struct store to derive the delete set — O(n) in the number
+   * of items. Use {@link fromDeleteSet} for the incremental path.
+   */
+  static fromDoc(doc: Doc): YjsVersion {
+    const sv = yjsEncodeStateVector(doc)
+    const snap = encodeSnapshot(yjsSnapshot(doc))
+    return new YjsVersion(sv, snap)
+  }
+
+  /**
+   * Construct a version from a `Y.Doc`'s state vector and an externally
+   * maintained delete set — the incremental path that avoids a struct
+   * store walk.
+   *
+   * @param doc  The live Y.Doc (for the state vector).
+   * @param ds   An accumulated delete set, kept in sync by merging
+   *             `transaction.deleteSet` on each transaction.
+   */
+  static fromDeleteSet(
+    doc: Doc,
+    ds: ReturnType<typeof import("yjs").createDeleteSet>,
+  ): YjsVersion {
+    const sv = yjsEncodeStateVector(doc)
+    const svMap = decodeStateVector(sv)
+    const snap = encodeSnapshot(createSnapshot(ds, svMap))
+    return new YjsVersion(sv, snap)
+  }
+
+  /**
+   * Serialize to a text-safe string.
+   *
+   * Format: `base64(sv) + "." + base64(snapshotBytes)`.
+   * The "." separator is unambiguous since base64 never contains ".".
    */
   serialize(): string {
-    return uint8ArrayToBase64(this.sv)
+    const svB64 = uint8ArrayToBase64(this.sv)
+    const snapB64 = uint8ArrayToBase64(this.snapshotBytes)
+    return svB64 + "." + snapB64
   }
 
   /**
-   * Compare with another version using version-vector partial order.
+   * Compare with another version using version-vector partial order,
+   * extended with delete-set equality checking.
    *
-   * Delegates to the shared `versionVectorCompare` utility after decoding
-   * both state vectors via `Y.decodeStateVector()`.
+   * 1. Decode both state vectors and compare via `versionVectorCompare`.
+   * 2. If the SV comparison yields anything other than "equal", return it.
+   * 3. If the SVs are equal, compare snapshot bytes for byte equality.
+   *    If they differ (same inserts, different deletes), return "concurrent"
+   *    — both sides may have tombstones the other lacks.
+   * 4. "equal" is returned only when BOTH the state vector AND the
+   *    delete set match.
    *
    * @throws If `other` is not a `YjsVersion`.
    */
@@ -98,18 +178,27 @@ export class YjsVersion implements Version {
     if (!(other instanceof YjsVersion)) {
       throw new Error("YjsVersion can only be compared with another YjsVersion")
     }
-    return versionVectorCompare(
+    const svResult = versionVectorCompare(
       decodeStateVector(this.sv),
       decodeStateVector(other.sv),
     )
+    if (svResult !== "equal") return svResult
+    // State vectors are equal — check if delete sets match via snapshot bytes.
+    return arraysEqual(this.snapshotBytes, other.snapshotBytes)
+      ? "equal"
+      : "concurrent"
   }
 
   /**
    * Greatest lower bound (lattice meet) of two Yjs versions.
    *
    * Decodes both state vectors, computes the component-wise minimum
-   * via the shared `versionVectorMeet` utility, and encodes the result
-   * back to a Yjs state vector.
+   * via `versionVectorMeet`, and encodes the result back to a Yjs
+   * state vector.
+   *
+   * The meet snapshot uses the meet SV with no delete-set information
+   * (conservative lower bound). meet() feeds into advance(), which Yjs
+   * does not support incrementally, so this is safe.
    *
    * @throws If `other` is not a `YjsVersion`.
    */
@@ -120,19 +209,32 @@ export class YjsVersion implements Version {
     const thisMap = decodeStateVector(this.sv)
     const otherMap = decodeStateVector(other.sv)
     const result = versionVectorMeet(thisMap, otherMap)
-    return new YjsVersion(encodeStateVector(result))
+    const meetSv = encodeStateVector(result)
+    // Conservative: meet snapshot uses only the meet SV (no delete set).
+    return new YjsVersion(meetSv)
   }
 
   /**
    * Parse a serialized YjsVersion string back into a YjsVersion.
    *
-   * The inverse of `serialize()`: base64 → `Uint8Array`.
+   * New format: `base64(sv) + "." + base64(snapshotBytes)`.
+   * Legacy format (no "."): `base64(sv)` only — constructed with
+   * `snapshotBytes` equal to the SV bytes. When compared against a
+   * new-format version with matching SVs, the differing snapshot bytes
+   * yield "concurrent", triggering a safe redundant sync push.
    */
   static parse(serialized: string): YjsVersion {
     if (serialized === "") {
       throw new Error("Invalid YjsVersion value: (empty string)")
     }
-    const bytes = base64ToUint8Array(serialized)
-    return new YjsVersion(bytes)
+    const dotIndex = serialized.indexOf(".")
+    if (dotIndex === -1) {
+      // Legacy format: SV-only (no delete set).
+      const bytes = base64ToUint8Array(serialized)
+      return new YjsVersion(bytes)
+    }
+    const sv = base64ToUint8Array(serialized.slice(0, dotIndex))
+    const snapshotBytes = base64ToUint8Array(serialized.slice(dotIndex + 1))
+    return new YjsVersion(sv, snapshotBytes)
   }
 }