@kyneta/yjs-schema 1.0.0 → 1.2.0

package/src/substrate.ts

@@ -8,25 +8,26 @@
 // The event bridge contract: wrapping a Y.Doc in a kyneta substrate
 // means subscribing to the kyneta doc observes ALL mutations to the
 // underlying Y.Doc, regardless of source (local kyneta writes,
-// importDelta, external Y.applyUpdate, external raw Yjs API mutations).
+// merge, external Y.applyUpdate, external raw Yjs API mutations).
 
 import type {
   ChangeBase,
   Path,
+  Reader,
+  Replica,
+  ReplicaFactory,
   Schema as SchemaNode,
-  StoreReader,
   Substrate,
   SubstrateFactory,
   SubstratePayload,
   WritableContext,
 } from "@kyneta/schema"
-import { buildWritableContext, executeBatch } from "@kyneta/schema"
+import { BACKING_DOC, buildWritableContext, executeBatch } from "@kyneta/schema"
 import * as Y from "yjs"
 import { applyChangeToYjs, eventsToOps } from "./change-mapping.js"
 import { ensureContainers } from "./populate.js"
-import { yjsStoreReader } from "./store-reader.js"
+import { yjsReader } from "./reader.js"
 import { YjsVersion } from "./version.js"
-import { registerYjsSubstrate } from "./yjs-escape.js"
 
 // ---------------------------------------------------------------------------
 // Origin tag — used to suppress echo from our own transactions
@@ -44,11 +45,11 @@ const KYNETA_ORIGIN = "kyneta-prepare"
  * This is the "bring your own doc" entry point. The user creates and
  * manages the Y.Doc (possibly via a Yjs provider); this function wraps
  * it with a schema-aware overlay providing typed reads, writes,
- * versioning, and export/import through the standard Substrate interface.
+ * versioning, and export/merge through the standard Substrate interface.
  *
  * **Event bridge contract:** A persistent `observeDeep` handler is
  * registered on the root Y.Map at construction time. All non-kyneta
- * mutations to the Y.Doc (imports, external local writes) are bridged
+ * mutations to the Y.Doc (merges, external local writes) are bridged
  * to the kyneta changefeed. Subscribing to the kyneta doc observes all
  * mutations regardless of source.
  *
@@ -71,8 +72,8 @@ export function createYjsSubstrate(
   // to be), and onFlush() skips transact/commit.
   let inOurTransaction = false
 
-  // Stashed origin from importDelta for the event bridge to pick up.
-  let pendingImportOrigin: string | undefined
+  // Stashed origin from merge for the event bridge to pick up.
+  let pendingMergeOrigin: string | undefined
 
   // Lazy-built WritableContext (same pattern as PlainSubstrate / LoroSubstrate).
   let cachedCtx: WritableContext | undefined
@@ -80,13 +81,15 @@ export function createYjsSubstrate(
   // The root Y.Map — all schema fields are children of this single map.
   const rootMap = doc.getMap("root")
 
-  // The StoreReader — live view over the Yjs shared type tree.
-  const reader: StoreReader = yjsStoreReader(doc, schema)
+  // The Reader — live view over the Yjs shared type tree.
+  const reader: Reader = yjsReader(doc, schema)
 
   // --- Substrate object ---
 
-  const substrate: Substrate<YjsVersion> = {
-    store: reader,
+  const substrate = {
+    [BACKING_DOC]: doc,
+
+    reader: reader,
 
     prepare(path: Path, change: ChangeBase): void {
       if (!inOurTransaction) {
@@ -98,7 +101,7 @@ export function createYjsSubstrate(
       // wrappedPrepare (changefeed layer) still buffers the op.
     },
 
-    onFlush(origin?: string): void {
+    onFlush(_origin?: string): void {
       if (!inOurTransaction && pendingChanges.length > 0) {
         // Local write: apply accumulated changes within a single
         // Yjs transaction tagged with our origin for echo suppression.
@@ -129,8 +132,21 @@ export function createYjsSubstrate(
       return new YjsVersion(Y.encodeStateVector(doc))
     },
 
-    exportSnapshot(): SubstratePayload {
+    baseVersion(): YjsVersion {
+      // Yjs substrate: base is always the initial state (no advance supported).
+      return new YjsVersion(new Uint8Array([0]))
+    },
+
+    advance(_to: YjsVersion): void {
+      throw new Error(
+        "advance() on a live Yjs substrate is not yet supported. " +
+          "Use advance() on a YjsReplica instead.",
+      )
+    },
+
+    exportEntirety(): SubstratePayload {
       return {
+        kind: "entirety",
         encoding: "binary",
         data: Y.encodeStateAsUpdate(doc),
       }
@@ -139,27 +155,28 @@ export function createYjsSubstrate(
     exportSince(since: YjsVersion): SubstratePayload | null {
       try {
         const bytes = Y.encodeStateAsUpdate(doc, since.sv)
-        return { encoding: "binary", data: bytes }
+        return { kind: "since", encoding: "binary", data: bytes }
       } catch {
         return null
       }
     },
 
-    importDelta(payload: SubstratePayload, origin?: string): void {
+    merge(payload: SubstratePayload, origin?: string): void {
       if (
         payload.encoding !== "binary" ||
         !(payload.data instanceof Uint8Array)
       ) {
         throw new Error(
-          "YjsSubstrate.importDelta only supports binary-encoded payloads",
+          "YjsSubstrate.merge expects binary-encoded payloads. " +
+            "If you recently switched CRDT backends, stale clients may be sending incompatible data.",
         )
       }
       // Stash origin for the event bridge to pick up
-      pendingImportOrigin = origin
+      pendingMergeOrigin = origin
       try {
         Y.applyUpdate(doc, payload.data, origin ?? "remote")
       } finally {
-        pendingImportOrigin = undefined
+        pendingMergeOrigin = undefined
       }
       // That's it — the observeDeep handler bridges events to the
       // changefeed via executeBatch.
@@ -175,18 +192,16 @@ export function createYjsSubstrate(
     }
 
     // Convert Yjs events → kyneta Ops
-    const ops = eventsToOps(events)
+    const ops = eventsToOps(events, schema)
     if (ops.length === 0) {
       return
     }
 
-    // Determine origin: prefer stashed kyneta origin (from importDelta),
+    // Determine origin: prefer stashed kyneta origin (from merge),
     // fall back to the transaction's origin if it's a string.
     const origin =
-      pendingImportOrigin ??
-      (typeof transaction.origin === "string"
-        ? transaction.origin
-        : undefined)
+      pendingMergeOrigin ??
+      (typeof transaction.origin === "string" ? transaction.origin : undefined)
 
     // Lazily ensure the context is built
     const ctx = substrate.context()
@@ -202,10 +217,7 @@ export function createYjsSubstrate(
     }
   })
 
-  // Register for the yjs() escape hatch
-  registerYjsSubstrate(substrate, doc)
-
-  return substrate
+  return substrate as Substrate<YjsVersion>
 }
 
 // ---------------------------------------------------------------------------
@@ -218,35 +230,167 @@ export function createYjsSubstrate(
  * - `create(schema)` — creates a fresh Y.Doc with empty containers
  *   matching the schema structure. No seed data — initial content
  *   should be applied via `change()` after construction.
- * - `fromSnapshot(payload, schema)` — creates a Y.Doc from a snapshot
+ * - `fromEntirety(payload, schema)` — creates a Y.Doc from an entirety
  *   payload, returns a substrate.
  * - `parseVersion(serialized)` — deserializes a YjsVersion.
  */
-export const yjsSubstrateFactory: SubstrateFactory<YjsVersion> = {
-  create(schema: SchemaNode): Substrate<YjsVersion> {
-    const doc = new Y.Doc()
-    ensureContainers(doc, schema)
-    return createYjsSubstrate(doc, schema)
+// ---------------------------------------------------------------------------
+// yjsReplicaFactory — ReplicaFactory<YjsVersion>
+// ---------------------------------------------------------------------------
+
+/**
+ * Schema-free replica factory for Yjs substrates.
+ *
+ * Constructs headless `Replica<YjsVersion>` instances backed by bare
+ * `Y.Doc`s — no schema walking, no container initialization, no
+ * Reader, no event bridge, no changefeed. Just the CRDT runtime
+ * with version tracking and export/merge.
+ *
+ * Used by conduit participants (stores, routing servers)
+ * that need to accumulate state, compute per-peer deltas, and compact
+ * storage without ever interpreting document fields.
+ */
+export function createYjsReplica(doc: Y.Doc): Replica<YjsVersion> {
+  let currentDoc = doc
+  let currentBase: YjsVersion = new YjsVersion(
+    Y.encodeStateVector(new Y.Doc()),
+  )
+
+  return {
+    get [BACKING_DOC]() {
+      return currentDoc
+    },
+
+    version(): YjsVersion {
+      return new YjsVersion(Y.encodeStateVector(currentDoc))
+    },
+
+    baseVersion(): YjsVersion {
+      return currentBase
+    },
+
+    advance(to: YjsVersion): void {
+      const baseCmp = currentBase.compare(to)
+      if (baseCmp === "ahead") {
+        throw new Error("advance(): target is behind base version")
+      }
+      const currentCmp = to.compare(this.version())
+      if (currentCmp === "ahead") {
+        throw new Error("advance(): target is ahead of current version")
+      }
+
+      // Yjs can only do full projection (to = version).
+      // For any to < version, it's a no-op — undershoot contract.
+      if (currentCmp !== "equal") return
+
+      // Full projection: create a new doc with current state, no history.
+      const update = Y.encodeStateAsUpdate(currentDoc)
+      const newDoc = new Y.Doc()
+      Y.applyUpdate(newDoc, update)
+      currentDoc = newDoc
+      currentBase = new YjsVersion(Y.encodeStateVector(currentDoc))
+    },
+
+    exportEntirety(): SubstratePayload {
+      return {
+        kind: "entirety",
+        encoding: "binary",
+        data: Y.encodeStateAsUpdate(currentDoc),
+      }
+    },
+
+    exportSince(since: YjsVersion): SubstratePayload | null {
+      try {
+        const bytes = Y.encodeStateAsUpdate(currentDoc, since.sv)
+        return { kind: "since", encoding: "binary", data: bytes }
+      } catch {
+        return null
+      }
+    },
+
+    merge(payload: SubstratePayload, _origin?: string): void {
+      if (
+        payload.encoding !== "binary" ||
+        !(payload.data instanceof Uint8Array)
+      ) {
+        throw new Error(
+          "YjsReplica.merge expects binary-encoded payloads. " +
+            "If you recently switched CRDT backends, stale clients may be sending incompatible data.",
+        )
+      }
+      Y.applyUpdate(currentDoc, payload.data)
+    },
+  } as Replica<YjsVersion>
+}
+
+export const yjsReplicaFactory: ReplicaFactory<YjsVersion> = {
+  replicaType: ["yjs", 1, 0] as const,
+
+  createEmpty(): Replica<YjsVersion> {
+    return createYjsReplica(new Y.Doc())
   },
 
-  fromSnapshot(
-    payload: SubstratePayload,
-    schema: SchemaNode,
-  ): Substrate<YjsVersion> {
+  fromEntirety(payload: SubstratePayload): Replica<YjsVersion> {
     if (
       payload.encoding !== "binary" ||
       !(payload.data instanceof Uint8Array)
     ) {
       throw new Error(
-        "YjsSubstrateFactory.fromSnapshot only supports binary-encoded payloads",
+        "YjsReplicaFactory.fromEntirety only supports binary-encoded payloads",
       )
     }
     const doc = new Y.Doc()
     Y.applyUpdate(doc, payload.data)
+    return createYjsReplica(doc)
+  },
+
+  parseVersion(serialized: string): YjsVersion {
+    return YjsVersion.parse(serialized)
+  },
+}
+
+// ---------------------------------------------------------------------------
+// yjsSubstrateFactory — SubstrateFactory<YjsVersion>
+// ---------------------------------------------------------------------------
+
+export const yjsSubstrateFactory: SubstrateFactory<YjsVersion> = {
+  replica: yjsReplicaFactory,
+
+  createReplica(): Replica<YjsVersion> {
+    // Default random clientID — safe for hydration (no local writes).
+    return createYjsReplica(new Y.Doc())
+  },
+
+  upgrade(
+    replica: Replica<YjsVersion>,
+    schema: SchemaNode,
+  ): Substrate<YjsVersion> {
+    const doc = (replica as any)[BACKING_DOC] as Y.Doc
+    // No identity injection for the standalone factory (no peerId).
+    // Conditional ensureContainers: skip fields that already exist
+    // from hydrated state.
+    ensureContainers(doc, schema, true)
+    return createYjsSubstrate(doc, schema)
+  },
+
+  create(schema: SchemaNode): Substrate<YjsVersion> {
+    // Fresh doc — unconditional ensureContainers (nothing to conflict with).
+    const doc = new Y.Doc()
+    ensureContainers(doc, schema)
     return createYjsSubstrate(doc, schema)
   },
 
+  fromEntirety(
+    payload: SubstratePayload,
+    schema: SchemaNode,
+  ): Substrate<YjsVersion> {
+    // Two-phase path: createReplica → merge → upgrade
+    const replica = this.createReplica()
+    replica.merge(payload)
+    return this.upgrade(replica, schema)
+  },
+
   parseVersion(serialized: string): YjsVersion {
     return YjsVersion.parse(serialized)
   },
-}
+}

package/src/sync.ts

@@ -1,17 +1,17 @@
 // sync — sync primitives for YjsSubstrate-backed documents.
 //
-// These functions provide version tracking, snapshot export, and delta
-// import for documents created via `createYjsDoc` or
-// `createYjsDocFromSnapshot`. They discover the substrate via the
+// These functions provide version tracking, entirety export, and merge
+// for documents created via `createYjsDoc` or
+// `createYjsDocFromEntirety`. They discover the substrate via the
 // module-scoped WeakMap in `create.ts`.
 //
 // Unlike PlainSubstrate's sync (which returns Op[] for deltas),
-// YjsSubstrate's sync uses binary SubstratePayload for both snapshots
+// YjsSubstrate's sync uses binary SubstratePayload for both entireties
 // and deltas — these are Yjs's native state-as-update bytes.
 
 import type { SubstratePayload } from "@kyneta/schema"
-import { YjsVersion } from "./version.js"
 import { getSubstrate } from "./create.js"
+import type { YjsVersion } from "./version.js"
 
 // ---------------------------------------------------------------------------
 // version — current YjsVersion
@@ -23,28 +23,28 @@ import { getSubstrate } from "./create.js"
  * Use `.serialize()` to get a text-safe string for embedding in HTML
  * meta tags, URL parameters, etc.
  *
- * @param doc - A document created by `createYjsDoc` or `createYjsDocFromSnapshot`.
- * @throws If `doc` was not created by `createYjsDoc` / `createYjsDocFromSnapshot`.
+ * @param doc - A document created by `createYjsDoc` or `createYjsDocFromEntirety`.
+ * @throws If `doc` was not created by `createYjsDoc` / `createYjsDocFromEntirety`.
  */
 export function version(doc: object): YjsVersion {
   return getSubstrate(doc).version()
 }
 
 // ---------------------------------------------------------------------------
-// exportSnapshot — full state for reconstruction
+// exportEntirety — full state for reconstruction
 // ---------------------------------------------------------------------------
 
 /**
- * Export the full substrate snapshot — sufficient for a new peer to
- * reconstruct an equivalent document via `createYjsDocFromSnapshot()`.
+ * Export the full substrate entirety — sufficient for a new peer to
+ * reconstruct an equivalent document via `createYjsDocFromEntirety()`.
  *
  * Returns a binary `SubstratePayload` (Yjs state-as-update bytes).
  *
- * @param doc - A document created by `createYjsDoc` or `createYjsDocFromSnapshot`.
- * @throws If `doc` was not created by `createYjsDoc` / `createYjsDocFromSnapshot`.
+ * @param doc - A document created by `createYjsDoc` or `createYjsDocFromEntirety`.
+ * @throws If `doc` was not created by `createYjsDoc` / `createYjsDocFromEntirety`.
  */
-export function exportSnapshot(doc: object): SubstratePayload {
-  return getSubstrate(doc).exportSnapshot()
+export function exportEntirety(doc: object): SubstratePayload {
+  return getSubstrate(doc).exportEntirety()
 }
 
 // ---------------------------------------------------------------------------
@@ -61,12 +61,12 @@ export function exportSnapshot(doc: object): SubstratePayload {
  * const v0 = version(docA)
  * change(docA, d => d.title.insert(0, "Hi"))
  * const delta = exportSince(docA, v0)
- * importDelta(docB, delta!)
+ * merge(docB, delta!)
  * ```
  *
- * @param doc - A document created by `createYjsDoc` or `createYjsDocFromSnapshot`.
+ * @param doc - A document created by `createYjsDoc` or `createYjsDocFromEntirety`.
  * @param since - The version to diff from.
- * @throws If `doc` was not created by `createYjsDoc` / `createYjsDocFromSnapshot`.
+ * @throws If `doc` was not created by `createYjsDoc` / `createYjsDocFromEntirety`.
  */
 export function exportSince(
   doc: object,
@@ -76,32 +76,32 @@ export function exportSince(
 }
 
 // ---------------------------------------------------------------------------
-// importDelta — apply a delta from another peer
+// merge — apply a delta from another peer
 // ---------------------------------------------------------------------------
 
 /**
  * Import a delta payload into a live document.
  *
  * The payload must have been produced by `exportSince()` or
- * `exportSnapshot()` on a compatible document.
+ * `exportEntirety()` on a compatible document.
  *
  * After import, the changefeed fires for all subscribers — the event
  * bridge handles this automatically.
  *
  * ```ts
  * const delta = exportSince(docA, sinceVersion)
- * importDelta(docB, delta!, "sync")
+ * merge(docB, delta!, "sync")
  * ```
  *
- * @param doc - A document created by `createYjsDoc` or `createYjsDocFromSnapshot`.
- * @param payload - The delta or snapshot payload to import.
+ * @param doc - A document created by `createYjsDoc` or `createYjsDocFromEntirety`.
+ * @param payload - The delta or entirety payload to merge.
  * @param origin - Optional provenance tag for the changeset.
- * @throws If `doc` was not created by `createYjsDoc` / `createYjsDocFromSnapshot`.
+ * @throws If `doc` was not created by `createYjsDoc` / `createYjsDocFromEntirety`.
  */
-export function importDelta(
+export function merge(
   doc: object,
   payload: SubstratePayload,
   origin?: string,
 ): void {
-  getSubstrate(doc).importDelta(payload, origin)
-}
+  getSubstrate(doc).merge(payload, origin)
+}

package/src/version.ts

@@ -12,6 +12,7 @@
 // decoded `Map<number, number>` (clientID → clock) maps ourselves.
 
 import type { Version } from "@kyneta/schema"
+import { versionVectorMeet } from "@kyneta/schema"
 import { decodeStateVector } from "yjs"
 
 // ---------------------------------------------------------------------------
@@ -35,6 +36,39 @@ function base64ToUint8Array(base64: string): Uint8Array {
   return bytes
 }
 
+// ---------------------------------------------------------------------------
+// State vector encoding — manual varint (unsigned LEB128)
+// ---------------------------------------------------------------------------
+
+/**
+ * Encode a state vector map to Yjs's binary state vector format.
+ *
+ * Yjs does not export `encodeStateVector(map)` — only `Y.encodeStateVector(doc)`
+ * which requires a full doc. This implements the same binary format directly:
+ * `[entryCount: varint, (clientId: varint, clock: varint)*]`
+ *
+ * Each value is encoded as an unsigned LEB128 varint.
+ */
+function encodeStateVector(map: Map<number, number>): Uint8Array {
+  const bytes: number[] = []
+
+  function writeVarUint(value: number): void {
+    while (value > 0x7f) {
+      bytes.push((value & 0x7f) | 0x80)
+      value >>>= 7
+    }
+    bytes.push(value & 0x7f)
+  }
+
+  writeVarUint(map.size)
+  for (const [clientId, clock] of map) {
+    writeVarUint(clientId)
+    writeVarUint(clock)
+  }
+
+  return new Uint8Array(bytes)
+}
+
 // ---------------------------------------------------------------------------
 // YjsVersion
 // ---------------------------------------------------------------------------
@@ -85,9 +119,7 @@ export class YjsVersion implements Version {
    */
   compare(other: Version): "behind" | "equal" | "ahead" | "concurrent" {
     if (!(other instanceof YjsVersion)) {
-      throw new Error(
-        "YjsVersion can only be compared with another YjsVersion",
-      )
+      throw new Error("YjsVersion can only be compared with another YjsVersion")
     }
 
     const thisMap = decodeStateVector(this.sv)
@@ -123,6 +155,27 @@ export class YjsVersion implements Version {
     return "equal"
   }
 
+  /**
+   * Greatest lower bound (lattice meet) of two Yjs versions.
+   *
+   * Decodes both state vectors, computes the component-wise minimum
+   * via the shared `versionVectorMeet` utility, and encodes the result
+   * back to a Yjs state vector.
+   *
+   * @throws If `other` is not a `YjsVersion`.
+   */
+  meet(other: Version): YjsVersion {
+    if (!(other instanceof YjsVersion)) {
+      throw new Error(
+        "YjsVersion can only be meet'd with another YjsVersion",
+      )
+    }
+    const thisMap = decodeStateVector(this.sv)
+    const otherMap = decodeStateVector(other.sv)
+    const result = versionVectorMeet(thisMap, otherMap)
+    return new YjsVersion(encodeStateVector(result))
+  }
+
   /**
    * Parse a serialized YjsVersion string back into a YjsVersion.
    *
@@ -135,4 +188,4 @@ export class YjsVersion implements Version {
     const bytes = base64ToUint8Array(serialized)
     return new YjsVersion(bytes)
   }
-}
+}

package/src/yjs-resolve.ts

@@ -14,9 +14,8 @@
 // Using a single root Y.Map enables one `observeDeep` call that
 // captures all mutations with correct relative paths.
 
-import { advanceSchema } from "@kyneta/schema"
-import type { Path, Segment } from "@kyneta/schema"
-import type { Schema as SchemaNode } from "@kyneta/schema"
+import type { Path, Schema as SchemaNode, Segment } from "@kyneta/schema"
+import { advanceSchema, KIND } from "@kyneta/schema"
 import * as Y from "yjs"
 
 // ---------------------------------------------------------------------------
@@ -35,10 +34,7 @@ import * as Y from "yjs"
  * @param current - The current position (a Yjs shared type or plain value)
  * @param segment - The path segment to follow
  */
-export function stepIntoYjs(
-  current: unknown,
-  segment: Segment,
-): unknown {
+export function stepIntoYjs(current: unknown, segment: Segment): unknown {
   const resolved = segment.resolve()
 
   if (current instanceof Y.Map) {
@@ -50,9 +46,7 @@ export function stepIntoYjs(
   }
 
   if (current instanceof Y.Text) {
-    throw new Error(
-      `yjs-resolve: cannot step into Y.Text`,
-    )
+    throw new Error(`yjs-resolve: cannot step into Y.Text`)
   }
 
   // Plain value — terminal, cannot step further
@@ -84,16 +78,6 @@ export function resolveYjsType(
   let current: unknown = rootMap
   let schema = rootSchema
 
-  // Unwrap the root annotation (e.g. annotated("doc", product))
-  // to reach the product schema whose fields are the root map's children.
-  let rootProduct = rootSchema
-  while (
-    rootProduct._kind === "annotated" &&
-    rootProduct.schema !== undefined
-  ) {
-    rootProduct = rootProduct.schema
-  }
-
   for (let i = 0; i < path.length; i++) {
     const seg = path.segments[i]!
     const nextSchema = advanceSchema(schema, seg)
@@ -105,4 +89,4 @@ export function resolveYjsType(
   }
 
   return current
-}
+}