@kyneta/yjs-schema 1.1.0 → 1.3.0

package/src/substrate.ts

@@ -22,12 +22,18 @@ import type {
   SubstratePayload,
   WritableContext,
 } from "@kyneta/schema"
-import { BACKING_DOC, buildWritableContext, executeBatch } from "@kyneta/schema"
+import {
+  BACKING_DOC,
+  buildWritableContext,
+  executeBatch,
+  KIND,
+} from "@kyneta/schema"
 import * as Y from "yjs"
 import { applyChangeToYjs, eventsToOps } from "./change-mapping.js"
 import { ensureContainers } from "./populate.js"
 import { yjsReader } from "./reader.js"
 import { YjsVersion } from "./version.js"
+import { resolveYjsType } from "./yjs-resolve.js"
 
 // ---------------------------------------------------------------------------
 // Origin tag — used to suppress echo from our own transactions
@@ -101,7 +107,7 @@ export function createYjsSubstrate(
       // wrappedPrepare (changefeed layer) still buffers the op.
     },
 
-    onFlush(origin?: string): void {
+    onFlush(_origin?: string): void {
       if (!inOurTransaction && pendingChanges.length > 0) {
         // Local write: apply accumulated changes within a single
         // Yjs transaction tagged with our origin for echo suppression.
@@ -124,6 +130,17 @@ export function createYjsSubstrate(
     context(): WritableContext {
       if (!cachedCtx) {
         cachedCtx = buildWritableContext(substrate)
+        // Attach nativeResolver — used by interpretImpl to set [NATIVE]
+        // on every ref. The resolver maps schema positions to Yjs shared types.
+        ;(cachedCtx as any).nativeResolver = (
+          nodeSchema: SchemaNode,
+          path: { segments: readonly unknown[] },
+        ) => {
+          if (path.segments.length === 0) return doc
+          if (nodeSchema[KIND] === "scalar" || nodeSchema[KIND] === "sum")
+            return undefined
+          return resolveYjsType(rootMap, schema, path as any)
+        }
       }
       return cachedCtx
     },
@@ -132,6 +149,18 @@ export function createYjsSubstrate(
       return new YjsVersion(Y.encodeStateVector(doc))
     },
 
+    baseVersion(): YjsVersion {
+      // Yjs substrate: base is always the initial state (no advance supported).
+      return new YjsVersion(new Uint8Array([0]))
+    },
+
+    advance(_to: YjsVersion): void {
+      throw new Error(
+        "advance() on a live Yjs substrate is not yet supported. " +
+          "Use advance() on a YjsReplica instead.",
+      )
+    },
+
     exportEntirety(): SubstratePayload {
       return {
         kind: "entirety",
@@ -180,7 +209,7 @@ export function createYjsSubstrate(
     }
 
     // Convert Yjs events → kyneta Ops
-    const ops = eventsToOps(events)
+    const ops = eventsToOps(events, schema)
     if (ops.length === 0) {
       return
     }
@@ -239,24 +268,55 @@ export function createYjsSubstrate(
  * storage without ever interpreting document fields.
  */
 export function createYjsReplica(doc: Y.Doc): Replica<YjsVersion> {
+  let currentDoc = doc
+  let currentBase: YjsVersion = new YjsVersion(Y.encodeStateVector(new Y.Doc()))
+
   return {
-    [BACKING_DOC]: doc,
+    get [BACKING_DOC]() {
+      return currentDoc
+    },
 
     version(): YjsVersion {
-      return new YjsVersion(Y.encodeStateVector(doc))
+      return new YjsVersion(Y.encodeStateVector(currentDoc))
+    },
+
+    baseVersion(): YjsVersion {
+      return currentBase
+    },
+
+    advance(to: YjsVersion): void {
+      const baseCmp = currentBase.compare(to)
+      if (baseCmp === "ahead") {
+        throw new Error("advance(): target is behind base version")
+      }
+      const currentCmp = to.compare(this.version())
+      if (currentCmp === "ahead") {
+        throw new Error("advance(): target is ahead of current version")
+      }
+
+      // Yjs can only do full projection (to = version).
+      // For any to < version, it's a no-op — undershoot contract.
+      if (currentCmp !== "equal") return
+
+      // Full projection: create a new doc with current state, no history.
+      const update = Y.encodeStateAsUpdate(currentDoc)
+      const newDoc = new Y.Doc()
+      Y.applyUpdate(newDoc, update)
+      currentDoc = newDoc
+      currentBase = new YjsVersion(Y.encodeStateVector(currentDoc))
     },
 
     exportEntirety(): SubstratePayload {
       return {
         kind: "entirety",
         encoding: "binary",
-        data: Y.encodeStateAsUpdate(doc),
+        data: Y.encodeStateAsUpdate(currentDoc),
       }
     },
 
     exportSince(since: YjsVersion): SubstratePayload | null {
       try {
-        const bytes = Y.encodeStateAsUpdate(doc, since.sv)
+        const bytes = Y.encodeStateAsUpdate(currentDoc, since.sv)
         return { kind: "since", encoding: "binary", data: bytes }
       } catch {
         return null
@@ -273,7 +333,7 @@ export function createYjsReplica(doc: Y.Doc): Replica<YjsVersion> {
             "If you recently switched CRDT backends, stale clients may be sending incompatible data.",
         )
       }
-      Y.applyUpdate(doc, payload.data)
+      Y.applyUpdate(currentDoc, payload.data)
     },
   } as Replica<YjsVersion>
 }

package/src/version.ts

@@ -12,27 +12,45 @@
 // decoded `Map<number, number>` (clientID → clock) maps ourselves.
 
 import type { Version } from "@kyneta/schema"
+import {
+  base64ToUint8Array,
+  uint8ArrayToBase64,
+  versionVectorCompare,
+  versionVectorMeet,
+} from "@kyneta/schema"
 import { decodeStateVector } from "yjs"
 
 // ---------------------------------------------------------------------------
-// Base64 helpers (platform-agnostic, no Node.js Buffer dependency)
+// State vector encoding — manual varint (unsigned LEB128)
 // ---------------------------------------------------------------------------
 
-function uint8ArrayToBase64(bytes: Uint8Array): string {
-  let binary = ""
-  for (let i = 0; i < bytes.length; i++) {
-    binary += String.fromCharCode(bytes[i]!)
+/**
+ * Encode a state vector map to Yjs's binary state vector format.
+ *
+ * Yjs does not export `encodeStateVector(map)` — only `Y.encodeStateVector(doc)`
+ * which requires a full doc. This implements the same binary format directly:
+ * `[entryCount: varint, (clientId: varint, clock: varint)*]`
+ *
+ * Each value is encoded as an unsigned LEB128 varint.
+ */
+function encodeStateVector(map: Map<number, number>): Uint8Array {
+  const bytes: number[] = []
+
+  function writeVarUint(value: number): void {
+    while (value > 0x7f) {
+      bytes.push((value & 0x7f) | 0x80)
+      value >>>= 7
+    }
+    bytes.push(value & 0x7f)
   }
-  return btoa(binary)
-}
 
-function base64ToUint8Array(base64: string): Uint8Array {
-  const binary = atob(base64)
-  const bytes = new Uint8Array(binary.length)
-  for (let i = 0; i < binary.length; i++) {
-    bytes[i] = binary.charCodeAt(i)
+  writeVarUint(map.size)
+  for (const [clientId, clock] of map) {
+    writeVarUint(clientId)
+    writeVarUint(clock)
   }
-  return bytes
+
+  return new Uint8Array(bytes)
 }
 
 // ---------------------------------------------------------------------------
@@ -71,54 +89,38 @@ export class YjsVersion implements Version {
   /**
    * Compare with another version using version-vector partial order.
    *
-   * Decodes both state vectors via `Y.decodeStateVector()` to get
-   * `Map<number, number>` (clientID → clock), then compares:
-   *
-   * - Collect the union of all client IDs from both maps.
-   * - For each client, compare clocks (missing client = clock 0).
-   * - If all clocks in `this` ≤ `other` and at least one strictly less → `"behind"`
-   * - If all clocks in `this` ≥ `other` and at least one strictly greater → `"ahead"`
-   * - If all clocks equal → `"equal"`
-   * - Otherwise → `"concurrent"`
+   * Delegates to the shared `versionVectorCompare` utility after decoding
+   * both state vectors via `Y.decodeStateVector()`.
    *
-   * Throws if `other` is not a `YjsVersion`.
+   * @throws If `other` is not a `YjsVersion`.
    */
   compare(other: Version): "behind" | "equal" | "ahead" | "concurrent" {
     if (!(other instanceof YjsVersion)) {
       throw new Error("YjsVersion can only be compared with another YjsVersion")
     }
+    return versionVectorCompare(
+      decodeStateVector(this.sv),
+      decodeStateVector(other.sv),
+    )
+  }
 
+  /**
+   * Greatest lower bound (lattice meet) of two Yjs versions.
+   *
+   * Decodes both state vectors, computes the component-wise minimum
+   * via the shared `versionVectorMeet` utility, and encodes the result
+   * back to a Yjs state vector.
+   *
+   * @throws If `other` is not a `YjsVersion`.
+   */
+  meet(other: Version): YjsVersion {
+    if (!(other instanceof YjsVersion)) {
+      throw new Error("YjsVersion can only be meet'd with another YjsVersion")
+    }
     const thisMap = decodeStateVector(this.sv)
     const otherMap = decodeStateVector(other.sv)
-
-    // Collect the union of all client IDs
-    const allClients = new Set<number>()
-    for (const id of thisMap.keys()) allClients.add(id)
-    for (const id of otherMap.keys()) allClients.add(id)
-
-    let hasLess = false
-    let hasGreater = false
-
-    for (const clientId of allClients) {
-      const thisClock = thisMap.get(clientId) ?? 0
-      const otherClock = otherMap.get(clientId) ?? 0
-
-      if (thisClock < otherClock) {
-        hasLess = true
-      }
-      if (thisClock > otherClock) {
-        hasGreater = true
-      }
-
-      // Early exit: if we've seen both less and greater, it's concurrent
-      if (hasLess && hasGreater) {
-        return "concurrent"
-      }
-    }
-
-    if (hasLess && !hasGreater) return "behind"
-    if (hasGreater && !hasLess) return "ahead"
-    return "equal"
+    const result = versionVectorMeet(thisMap, otherMap)
+    return new YjsVersion(encodeStateVector(result))
   }
 
   /**

package/src/yjs-resolve.ts

@@ -78,16 +78,6 @@ export function resolveYjsType(
   let current: unknown = rootMap
   let schema = rootSchema
 
-  // Unwrap the root annotation (e.g. annotated("doc", product))
-  // to reach the product schema whose fields are the root map's children.
-  let rootProduct = rootSchema
-  while (
-    rootProduct._kind === "annotated" &&
-    rootProduct.schema !== undefined
-  ) {
-    rootProduct = rootProduct.schema
-  }
-
   for (let i = 0; i < path.length; i++) {
     const seg = path.segments[i]!
     const nextSchema = advanceSchema(schema, seg)

package/src/create.ts

@@ -1,177 +0,0 @@
-// create — batteries-included document construction backed by YjsSubstrate.
-//
-// Provides `createYjsDoc` and `createYjsDocFromEntirety` functions that
-// hide the interpret pipeline and layer composition behind a single call.
-//
-// Internally tracks substrates via a module-scoped WeakMap so that sync
-// primitives (`version`, `exportEntirety`, `merge` in sync.ts)
-// can retrieve the substrate from just a doc ref.
-//
-// `getSubstrate` is exported for use by `sync.ts` but is NOT re-exported
-// from the barrel (`index.ts`). It is an internal cross-module helper.
-//
-// Two forms for `createYjsDoc`:
-//   createYjsDoc(schema, yjsDoc)    — "bring your own doc" (wrap existing)
-//   createYjsDoc(schema)            — create a fresh empty Y.Doc
-
-import type {
-  Ref,
-  Schema as SchemaType,
-  Substrate,
-  SubstratePayload,
-} from "@kyneta/schema"
-import {
-  changefeed,
-  interpret,
-  readable,
-  registerSubstrate,
-  writable,
-} from "@kyneta/schema"
-import type * as Y from "yjs"
-import { createYjsSubstrate, yjsSubstrateFactory } from "./substrate.js"
-import type { YjsVersion } from "./version.js"
-
-// ---------------------------------------------------------------------------
-// Substrate tracking (module-scoped)
-// ---------------------------------------------------------------------------
-
-const substrates = new WeakMap<object, Substrate<YjsVersion>>()
-
-/**
- * Retrieve the substrate associated with a doc created by `createYjsDoc`
- * or `createYjsDocFromEntirety`.
- *
- * Exported for `sync.ts` — NOT re-exported from the barrel.
- *
- * @throws If `doc` was not created by `createYjsDoc` / `createYjsDocFromEntirety`.
- */
-export function getSubstrate(doc: object): Substrate<YjsVersion> {
-  const s = substrates.get(doc)
-  if (!s) {
-    throw new Error(
-      "version/exportEntirety/merge called on an object without a YjsSubstrate. " +
-        "Use a doc created by createYjsDoc() or createYjsDocFromEntirety().",
-    )
-  }
-  return s
-}
-
-// ---------------------------------------------------------------------------
-// registerDoc — internal helper (interpret + WeakMap registration)
-// ---------------------------------------------------------------------------
-
-function registerDoc(
-  schema: SchemaType,
-  substrate: Substrate<YjsVersion>,
-): any {
-  // The `as any` on the builder avoids TS2589 — interpret's fluent API
-  // produces deeply recursive types when S is the abstract SchemaType.
-  // The public createYjsDoc/createYjsDocFromEntirety signatures provide
-  // the correct Ref<S> return type via interface call signature patterns.
-  const doc: any = (interpret as any)(schema, substrate.context())
-    .with(readable)
-    .with(writable)
-    .with(changefeed)
-    .done()
-  substrates.set(doc, substrate)
-  // Also register in the general unwrap() registry so that the
-  // yjs() escape hatch can discover the substrate from the ref.
-  registerSubstrate(doc, substrate)
-  return doc
-}
-
-// ---------------------------------------------------------------------------
-// isYDoc — runtime check for Y.Doc
-// ---------------------------------------------------------------------------
-
-function isYDoc(value: unknown): value is Y.Doc {
-  return (
-    value !== null &&
-    value !== undefined &&
-    typeof value === "object" &&
-    "getMap" in value &&
-    "getText" in value &&
-    "getArray" in value &&
-    "transact" in value &&
-    typeof (value as any).transact === "function" &&
-    // Y.Doc has clientID; distinguish from other objects
-    "clientID" in value &&
-    typeof (value as any).clientID === "number"
-  )
-}
-
-// ---------------------------------------------------------------------------
-// createYjsDoc
-// ---------------------------------------------------------------------------
-
-// Interface call signature avoids TS2589 on Ref<S> when S is generic.
-
-/**
- * Create a live Yjs-backed document.
- *
- * **Form 1 — bring your own doc:**
- * ```ts
- * const yjsDoc = new Y.Doc()
- * const doc = createYjsDoc(mySchema, yjsDoc)
- * ```
- *
- * **Form 2 — fresh empty doc:**
- * ```ts
- * const doc = createYjsDoc(mySchema)
- *
- * // Apply initial content via change():
- * change(doc, d => {
- *   d.title.insert(0, "Hello")
- *   d.items.push({ name: "First item" })
- * })
- * ```
- *
- * Returns a full-stack `Ref<S>` — callable, navigable, writable,
- * transactable, and observable. Backed by a `YjsSubstrate` with
- * CRDT collaboration support.
- *
- * The returned ref observes **all** mutations to the underlying Y.Doc,
- * regardless of source (local kyneta writes, merge, external
- * `Y.applyUpdate()`, external raw Yjs API mutations).
- *
- * @param schema - The schema describing the document structure.
- * @param doc - Optional `Y.Doc` instance to wrap. If omitted, a fresh
- *   empty Y.Doc is created with containers matching the schema.
- */
-type CreateYjsDoc = <S extends SchemaType>(schema: S, doc?: Y.Doc) => Ref<S>
-
-export const createYjsDoc: CreateYjsDoc = (schema, doc) => {
-  if (doc !== undefined && isYDoc(doc)) {
-    // Bring your own doc — wrap the existing Y.Doc
-    return registerDoc(schema, createYjsSubstrate(doc, schema))
-  }
-  // Fresh empty doc
-  return registerDoc(schema, yjsSubstrateFactory.create(schema))
-}
-
-// ---------------------------------------------------------------------------
-// createYjsDocFromEntirety
-// ---------------------------------------------------------------------------
-
-type CreateYjsDocFromEntirety = <S extends SchemaType>(
-  schema: S,
-  payload: SubstratePayload,
-) => Ref<S>
-
-/**
- * Reconstruct a live Yjs-backed document from a substrate entirety payload.
- *
- * The payload must have been produced by `exportEntirety()` on a
- * compatible document. This is the entry point for SSR hydration
- * and reconnection past log compaction.
- *
- * ```ts
- * const payload = exportEntirety(docA)
- * const docB = createYjsDocFromEntirety(MySchema, payload)
- * // docB has the same state as docA at the time of export
- * ```
- */
-export const createYjsDocFromEntirety: CreateYjsDocFromEntirety = (
-  schema,
-  payload,
-) => registerDoc(schema, yjsSubstrateFactory.fromEntirety(payload, schema))

package/src/sync.ts

@@ -1,107 +0,0 @@
-// sync — sync primitives for YjsSubstrate-backed documents.
-//
-// These functions provide version tracking, entirety export, and merge
-// for documents created via `createYjsDoc` or
-// `createYjsDocFromEntirety`. They discover the substrate via the
-// module-scoped WeakMap in `create.ts`.
-//
-// Unlike PlainSubstrate's sync (which returns Op[] for deltas),
-// YjsSubstrate's sync uses binary SubstratePayload for both entireties
-// and deltas — these are Yjs's native state-as-update bytes.
-
-import type { SubstratePayload } from "@kyneta/schema"
-import { getSubstrate } from "./create.js"
-import type { YjsVersion } from "./version.js"
-
-// ---------------------------------------------------------------------------
-// version — current YjsVersion
-// ---------------------------------------------------------------------------
-
-/**
- * Current version as a `YjsVersion` (wrapping a Yjs state vector).
- *
- * Use `.serialize()` to get a text-safe string for embedding in HTML
- * meta tags, URL parameters, etc.
- *
- * @param doc - A document created by `createYjsDoc` or `createYjsDocFromEntirety`.
- * @throws If `doc` was not created by `createYjsDoc` / `createYjsDocFromEntirety`.
- */
-export function version(doc: object): YjsVersion {
-  return getSubstrate(doc).version()
-}
-
-// ---------------------------------------------------------------------------
-// exportEntirety — full state for reconstruction
-// ---------------------------------------------------------------------------
-
-/**
- * Export the full substrate entirety — sufficient for a new peer to
- * reconstruct an equivalent document via `createYjsDocFromEntirety()`.
- *
- * Returns a binary `SubstratePayload` (Yjs state-as-update bytes).
- *
- * @param doc - A document created by `createYjsDoc` or `createYjsDocFromEntirety`.
- * @throws If `doc` was not created by `createYjsDoc` / `createYjsDocFromEntirety`.
- */
-export function exportEntirety(doc: object): SubstratePayload {
-  return getSubstrate(doc).exportEntirety()
-}
-
-// ---------------------------------------------------------------------------
-// exportSince — delta since a version
-// ---------------------------------------------------------------------------
-
-/**
- * Export a delta payload containing all changes since the given version.
- *
- * Returns a binary `SubstratePayload` (Yjs update bytes), or `null`
- * if the delta cannot be computed.
- *
- * ```ts
- * const v0 = version(docA)
- * change(docA, d => d.title.insert(0, "Hi"))
- * const delta = exportSince(docA, v0)
- * merge(docB, delta!)
- * ```
- *
- * @param doc - A document created by `createYjsDoc` or `createYjsDocFromEntirety`.
- * @param since - The version to diff from.
- * @throws If `doc` was not created by `createYjsDoc` / `createYjsDocFromEntirety`.
- */
-export function exportSince(
-  doc: object,
-  since: YjsVersion,
-): SubstratePayload | null {
-  return getSubstrate(doc).exportSince(since)
-}
-
-// ---------------------------------------------------------------------------
-// merge — apply a delta from another peer
-// ---------------------------------------------------------------------------
-
-/**
- * Import a delta payload into a live document.
- *
- * The payload must have been produced by `exportSince()` or
- * `exportEntirety()` on a compatible document.
- *
- * After import, the changefeed fires for all subscribers — the event
- * bridge handles this automatically.
- *
- * ```ts
- * const delta = exportSince(docA, sinceVersion)
- * merge(docB, delta!, "sync")
- * ```
- *
- * @param doc - A document created by `createYjsDoc` or `createYjsDocFromEntirety`.
- * @param payload - The delta or entirety payload to merge.
- * @param origin - Optional provenance tag for the changeset.
- * @throws If `doc` was not created by `createYjsDoc` / `createYjsDocFromEntirety`.
- */
-export function merge(
-  doc: object,
-  payload: SubstratePayload,
-  origin?: string,
-): void {
-  getSubstrate(doc).merge(payload, origin)
-}

package/src/yjs-escape.ts

@@ -1,84 +0,0 @@
-// yjs-escape — Yjs-specific escape hatch for accessing the Y.Doc
-// backing a ref.
-//
-// `yjs(ref)` returns the `Y.Doc` backing a root document ref.
-//
-// The substrate exposes its backing Y.Doc via the `BACKING_DOC` symbol
-// (from `@kyneta/schema`). The `yjs()` function uses `unwrap()` to get
-// the substrate, then reads `[BACKING_DOC]` to get the Y.Doc.
-//
-// This two-step approach (ref → substrate → Y.Doc) avoids duplicating
-// the ref-tracking WeakMap and composes cleanly with the general
-// `unwrap()` escape hatch.
-//
-// Context: jj:smmulzkm (BACKING_DOC replaces WeakMap + registerYjsSubstrate)
-//
-// Usage:
-//   import { yjs } from "@kyneta/yjs-schema"
-//
-//   const doc = exchange.get("my-doc", TodoDoc)
-//   const yjsDoc = yjs(doc)  // Y.Doc
-//   yjsDoc.getMap("root").toJSON()  // raw Yjs inspection
-
-import { BACKING_DOC, unwrap } from "@kyneta/schema"
-import type { Doc as YDoc } from "yjs"
-
-// ---------------------------------------------------------------------------
-// yjs — Yjs-specific escape hatch
-// ---------------------------------------------------------------------------
-
-/**
- * Returns the `Y.Doc` backing the given ref.
- *
- * This is the Yjs-specific escape hatch for accessing substrate-level
- * capabilities: raw Yjs API, y-prosemirror/y-codemirror bindings,
- * undo manager, awareness protocol, Yjs providers (y-websocket,
- * y-indexeddb, y-webrtc, Hocuspocus, Liveblocks), etc.
- *
- * Currently supports root document refs only. Child-level resolution
- * (e.g. `yjs(doc.title)` → `Y.Text`) is future work.
- *
- * @param ref - A root document ref backed by a Yjs substrate
- * @returns The `Y.Doc` backing the ref
- * @throws If the ref is not backed by a Yjs substrate
- *
- * @example
- * ```ts
- * import { yjs } from "@kyneta/yjs-schema"
- *
- * const doc = exchange.get("my-doc", TodoDoc)
- * const yjsDoc = yjs(doc)
- * console.log(yjsDoc.getMap("root").toJSON())  // raw state
- * console.log(yjsDoc.clientID)                  // client ID
- * ```
- */
-export function yjs(ref: object): YDoc {
-  let substrate: any
-  try {
-    substrate = unwrap(ref)
-  } catch {
-    throw new Error(
-      "yjs() requires a ref backed by a Yjs substrate. " +
-        "Use a doc created by exchange.get() with a bindYjs() schema, " +
-        "or by createYjsDoc().",
-    )
-  }
-
-  const doc = substrate[BACKING_DOC]
-  // Duck-type check: Y.Doc has getMap, encodeStateVector-compatible API,
-  // and a numeric clientID. A PlainState (plain object) or LoroDoc would
-  // not have getMap as a function.
-  if (
-    !doc ||
-    typeof doc !== "object" ||
-    typeof (doc as any).getMap !== "function" ||
-    typeof (doc as any).clientID !== "number"
-  ) {
-    throw new Error(
-      "yjs() requires a ref backed by a Yjs substrate. " +
-        "The ref has a substrate but it is not a Yjs substrate. " +
-        "Use a doc created with a bindYjs() schema or createYjsDoc().",
-    )
-  }
-  return doc as YDoc
-}