@kyneta/yjs-schema 1.0.0

package/src/substrate.ts

@@ -0,0 +1,252 @@
+// substrate — YjsSubstrate implementation.
+//
+// Implements Substrate<YjsVersion> with:
+// - Imperative local writes (prepare accumulates, onFlush applies in transact)
+// - Persistent observeDeep event bridge for external changes
+// - Single re-entrancy guard + transaction.origin check
+//
+// The event bridge contract: wrapping a Y.Doc in a kyneta substrate
+// means subscribing to the kyneta doc observes ALL mutations to the
+// underlying Y.Doc, regardless of source (local kyneta writes,
+// importDelta, external Y.applyUpdate, external raw Yjs API mutations).
+
+import type {
+  ChangeBase,
+  Path,
+  Schema as SchemaNode,
+  StoreReader,
+  Substrate,
+  SubstrateFactory,
+  SubstratePayload,
+  WritableContext,
+} from "@kyneta/schema"
+import { buildWritableContext, executeBatch } from "@kyneta/schema"
+import * as Y from "yjs"
+import { applyChangeToYjs, eventsToOps } from "./change-mapping.js"
+import { ensureContainers } from "./populate.js"
+import { yjsStoreReader } from "./store-reader.js"
+import { YjsVersion } from "./version.js"
+import { registerYjsSubstrate } from "./yjs-escape.js"
+
+// ---------------------------------------------------------------------------
+// Origin tag — used to suppress echo from our own transactions
+// ---------------------------------------------------------------------------
+
+const KYNETA_ORIGIN = "kyneta-prepare"
+
+// ---------------------------------------------------------------------------
+// createYjsSubstrate — wrap a user-provided Y.Doc
+// ---------------------------------------------------------------------------
+
+/**
+ * Creates a `Substrate<YjsVersion>` wrapping a user-provided Y.Doc.
+ *
+ * This is the "bring your own doc" entry point. The user creates and
+ * manages the Y.Doc (possibly via a Yjs provider); this function wraps
+ * it with a schema-aware overlay providing typed reads, writes,
+ * versioning, and export/import through the standard Substrate interface.
+ *
+ * **Event bridge contract:** A persistent `observeDeep` handler is
+ * registered on the root Y.Map at construction time. All non-kyneta
+ * mutations to the Y.Doc (imports, external local writes) are bridged
+ * to the kyneta changefeed. Subscribing to the kyneta doc observes all
+ * mutations regardless of source.
+ *
+ * @param doc - The Y.Doc to wrap. The substrate does NOT own the doc;
+ *   the caller is responsible for its lifecycle.
+ * @param schema - The root schema for the document.
+ */
+export function createYjsSubstrate(
+  doc: Y.Doc,
+  schema: SchemaNode,
+): Substrate<YjsVersion> {
+  // --- Closure-scoped state ---
+
+  // Accumulated changes from prepare(), drained by onFlush().
+  const pendingChanges: Array<{ path: Path; change: ChangeBase }> = []
+
+  // Re-entrancy guard: set true around our doc.transact() in onFlush
+  // AND around executeBatch in the event bridge. When true, prepare()
+  // skips Yjs-side work (changes are already applied by Yjs or about
+  // to be), and onFlush() skips transact/commit.
+  let inOurTransaction = false
+
+  // Stashed origin from importDelta for the event bridge to pick up.
+  let pendingImportOrigin: string | undefined
+
+  // Lazy-built WritableContext (same pattern as PlainSubstrate / LoroSubstrate).
+  let cachedCtx: WritableContext | undefined
+
+  // The root Y.Map — all schema fields are children of this single map.
+  const rootMap = doc.getMap("root")
+
+  // The StoreReader — live view over the Yjs shared type tree.
+  const reader: StoreReader = yjsStoreReader(doc, schema)
+
+  // --- Substrate object ---
+
+  const substrate: Substrate<YjsVersion> = {
+    store: reader,
+
+    prepare(path: Path, change: ChangeBase): void {
+      if (!inOurTransaction) {
+        // Local write: accumulate for flush.
+        // No Yjs side effects — mutations happen at flush time.
+        pendingChanges.push({ path, change })
+      }
+      // During event handler replay: no-op on Yjs side.
+      // wrappedPrepare (changefeed layer) still buffers the op.
+    },
+
+    onFlush(origin?: string): void {
+      if (!inOurTransaction && pendingChanges.length > 0) {
+        // Local write: apply accumulated changes within a single
+        // Yjs transaction tagged with our origin for echo suppression.
+        inOurTransaction = true
+        try {
+          doc.transact(() => {
+            for (const { path, change } of pendingChanges) {
+              applyChangeToYjs(rootMap, schema, path, change)
+            }
+          }, KYNETA_ORIGIN)
+          pendingChanges.length = 0
+        } finally {
+          inOurTransaction = false
+        }
+      }
+      // During event handler replay: no-op on Yjs side.
+      // wrappedFlush (changefeed layer) still delivers notifications.
+    },
+
+    context(): WritableContext {
+      if (!cachedCtx) {
+        cachedCtx = buildWritableContext(substrate)
+      }
+      return cachedCtx
+    },
+
+    version(): YjsVersion {
+      return new YjsVersion(Y.encodeStateVector(doc))
+    },
+
+    exportSnapshot(): SubstratePayload {
+      return {
+        encoding: "binary",
+        data: Y.encodeStateAsUpdate(doc),
+      }
+    },
+
+    exportSince(since: YjsVersion): SubstratePayload | null {
+      try {
+        const bytes = Y.encodeStateAsUpdate(doc, since.sv)
+        return { encoding: "binary", data: bytes }
+      } catch {
+        return null
+      }
+    },
+
+    importDelta(payload: SubstratePayload, origin?: string): void {
+      if (
+        payload.encoding !== "binary" ||
+        !(payload.data instanceof Uint8Array)
+      ) {
+        throw new Error(
+          "YjsSubstrate.importDelta only supports binary-encoded payloads",
+        )
+      }
+      // Stash origin for the event bridge to pick up
+      pendingImportOrigin = origin
+      try {
+        Y.applyUpdate(doc, payload.data, origin ?? "remote")
+      } finally {
+        pendingImportOrigin = undefined
+      }
+      // That's it — the observeDeep handler bridges events to the
+      // changefeed via executeBatch.
+    },
+  }
+
+  // --- Event bridge (registered once at construction) ---
+
+  rootMap.observeDeep((events, transaction) => {
+    // Ignore our own transactions (changefeed already captured via wrappedPrepare)
+    if (transaction.origin === KYNETA_ORIGIN) {
+      return
+    }
+
+    // Convert Yjs events → kyneta Ops
+    const ops = eventsToOps(events)
+    if (ops.length === 0) {
+      return
+    }
+
+    // Determine origin: prefer stashed kyneta origin (from importDelta),
+    // fall back to the transaction's origin if it's a string.
+    const origin =
+      pendingImportOrigin ??
+      (typeof transaction.origin === "string"
+        ? transaction.origin
+        : undefined)
+
+    // Lazily ensure the context is built
+    const ctx = substrate.context()
+
+    // Feed through executeBatch for changefeed delivery.
+    // The inOurTransaction guard prevents prepare/onFlush from doing
+    // Yjs-side work — the changes are already applied by Yjs.
+    inOurTransaction = true
+    try {
+      executeBatch(ctx, ops, origin)
+    } finally {
+      inOurTransaction = false
+    }
+  })
+
+  // Register for the yjs() escape hatch
+  registerYjsSubstrate(substrate, doc)
+
+  return substrate
+}
+
+// ---------------------------------------------------------------------------
+// yjsSubstrateFactory — SubstrateFactory<YjsVersion>
+// ---------------------------------------------------------------------------
+
+/**
+ * Factory for constructing Yjs-backed substrates.
+ *
+ * - `create(schema)` — creates a fresh Y.Doc with empty containers
+ *   matching the schema structure. No seed data — initial content
+ *   should be applied via `change()` after construction.
+ * - `fromSnapshot(payload, schema)` — creates a Y.Doc from a snapshot
+ *   payload, returns a substrate.
+ * - `parseVersion(serialized)` — deserializes a YjsVersion.
+ */
+export const yjsSubstrateFactory: SubstrateFactory<YjsVersion> = {
+  create(schema: SchemaNode): Substrate<YjsVersion> {
+    const doc = new Y.Doc()
+    ensureContainers(doc, schema)
+    return createYjsSubstrate(doc, schema)
+  },
+
+  fromSnapshot(
+    payload: SubstratePayload,
+    schema: SchemaNode,
+  ): Substrate<YjsVersion> {
+    if (
+      payload.encoding !== "binary" ||
+      !(payload.data instanceof Uint8Array)
+    ) {
+      throw new Error(
+        "YjsSubstrateFactory.fromSnapshot only supports binary-encoded payloads",
+      )
+    }
+    const doc = new Y.Doc()
+    Y.applyUpdate(doc, payload.data)
+    return createYjsSubstrate(doc, schema)
+  },
+
+  parseVersion(serialized: string): YjsVersion {
+    return YjsVersion.parse(serialized)
+  },
+}

package/src/sync.ts

@@ -0,0 +1,107 @@
+// sync — sync primitives for YjsSubstrate-backed documents.
+//
+// These functions provide version tracking, snapshot export, and delta
+// import for documents created via `createYjsDoc` or
+// `createYjsDocFromSnapshot`. They discover the substrate via the
+// module-scoped WeakMap in `create.ts`.
+//
+// Unlike PlainSubstrate's sync (which returns Op[] for deltas),
+// YjsSubstrate's sync uses binary SubstratePayload for both snapshots
+// and deltas — these are Yjs's native state-as-update bytes.
+
+import type { SubstratePayload } from "@kyneta/schema"
+import { YjsVersion } from "./version.js"
+import { getSubstrate } from "./create.js"
+
+// ---------------------------------------------------------------------------
+// version — current YjsVersion
+// ---------------------------------------------------------------------------
+
+/**
+ * Current version as a `YjsVersion` (wrapping a Yjs state vector).
+ *
+ * Use `.serialize()` to get a text-safe string for embedding in HTML
+ * meta tags, URL parameters, etc.
+ *
+ * @param doc - A document created by `createYjsDoc` or `createYjsDocFromSnapshot`.
+ * @throws If `doc` was not created by `createYjsDoc` / `createYjsDocFromSnapshot`.
+ */
+export function version(doc: object): YjsVersion {
+  return getSubstrate(doc).version()
+}
+
+// ---------------------------------------------------------------------------
+// exportSnapshot — full state for reconstruction
+// ---------------------------------------------------------------------------
+
+/**
+ * Export the full substrate snapshot — sufficient for a new peer to
+ * reconstruct an equivalent document via `createYjsDocFromSnapshot()`.
+ *
+ * Returns a binary `SubstratePayload` (Yjs state-as-update bytes).
+ *
+ * @param doc - A document created by `createYjsDoc` or `createYjsDocFromSnapshot`.
+ * @throws If `doc` was not created by `createYjsDoc` / `createYjsDocFromSnapshot`.
+ */
+export function exportSnapshot(doc: object): SubstratePayload {
+  return getSubstrate(doc).exportSnapshot()
+}
+
+// ---------------------------------------------------------------------------
+// exportSince — delta since a version
+// ---------------------------------------------------------------------------
+
+/**
+ * Export a delta payload containing all changes since the given version.
+ *
+ * Returns a binary `SubstratePayload` (Yjs update bytes), or `null`
+ * if the delta cannot be computed.
+ *
+ * ```ts
+ * const v0 = version(docA)
+ * change(docA, d => d.title.insert(0, "Hi"))
+ * const delta = exportSince(docA, v0)
+ * importDelta(docB, delta!)
+ * ```
+ *
+ * @param doc - A document created by `createYjsDoc` or `createYjsDocFromSnapshot`.
+ * @param since - The version to diff from.
+ * @throws If `doc` was not created by `createYjsDoc` / `createYjsDocFromSnapshot`.
+ */
+export function exportSince(
+  doc: object,
+  since: YjsVersion,
+): SubstratePayload | null {
+  return getSubstrate(doc).exportSince(since)
+}
+
+// ---------------------------------------------------------------------------
+// importDelta — apply a delta from another peer
+// ---------------------------------------------------------------------------
+
+/**
+ * Import a delta payload into a live document.
+ *
+ * The payload must have been produced by `exportSince()` or
+ * `exportSnapshot()` on a compatible document.
+ *
+ * After import, the changefeed fires for all subscribers — the event
+ * bridge handles this automatically.
+ *
+ * ```ts
+ * const delta = exportSince(docA, sinceVersion)
+ * importDelta(docB, delta!, "sync")
+ * ```
+ *
+ * @param doc - A document created by `createYjsDoc` or `createYjsDocFromSnapshot`.
+ * @param payload - The delta or snapshot payload to import.
+ * @param origin - Optional provenance tag for the changeset.
+ * @throws If `doc` was not created by `createYjsDoc` / `createYjsDocFromSnapshot`.
+ */
+export function importDelta(
+  doc: object,
+  payload: SubstratePayload,
+  origin?: string,
+): void {
+  getSubstrate(doc).importDelta(payload, origin)
+}

package/src/version.ts

@@ -0,0 +1,138 @@
+// YjsVersion — Version implementation wrapping Yjs state vectors.
+//
+// Yjs state vectors (`Y.encodeStateVector(doc)`) are the complete peer
+// state used for sync diffing — matching the semantics of kyneta's
+// Version interface.
+//
+// Serialization uses base64-encoded bytes for text-safe embedding in
+// HTML meta tags, script tags, etc.
+//
+// Yjs does not export a state vector comparison function, so we
+// implement standard version-vector partial-order comparison over
+// decoded `Map<number, number>` (clientID → clock) maps ourselves.
+
+import type { Version } from "@kyneta/schema"
+import { decodeStateVector } from "yjs"
+
+// ---------------------------------------------------------------------------
+// Base64 helpers (platform-agnostic, no Node.js Buffer dependency)
+// ---------------------------------------------------------------------------
+
+function uint8ArrayToBase64(bytes: Uint8Array): string {
+  let binary = ""
+  for (let i = 0; i < bytes.length; i++) {
+    binary += String.fromCharCode(bytes[i]!)
+  }
+  return btoa(binary)
+}
+
+function base64ToUint8Array(base64: string): Uint8Array {
+  const binary = atob(base64)
+  const bytes = new Uint8Array(binary.length)
+  for (let i = 0; i < binary.length; i++) {
+    bytes[i] = binary.charCodeAt(i)
+  }
+  return bytes
+}
+
+// ---------------------------------------------------------------------------
+// YjsVersion
+// ---------------------------------------------------------------------------
+
+/**
+ * A Version wrapping a Yjs state vector.
+ *
+ * State vectors track the complete peer state — which operations from
+ * each client have been observed. This is the right abstraction for sync
+ * diffing: `exportSince(version)` uses the state vector to compute the
+ * minimal update payload via `Y.encodeStateAsUpdate(doc, sv)`.
+ *
+ * `serialize()` encodes to base64 for text-safe embedding.
+ * `compare()` decodes both state vectors and performs standard
+ * version-vector partial-order comparison over the client-clock maps.
+ */
+export class YjsVersion implements Version {
+  readonly sv: Uint8Array
+
+  constructor(sv: Uint8Array) {
+    this.sv = sv
+  }
+
+  /**
+   * Serialize the state vector to a base64 string.
+   *
+   * The encoding is: raw state vector bytes → base64.
+   * This is text-safe for embedding in HTML meta tags, URL parameters, etc.
+   */
+  serialize(): string {
+    return uint8ArrayToBase64(this.sv)
+  }
+
+  /**
+   * Compare with another version using version-vector partial order.
+   *
+   * Decodes both state vectors via `Y.decodeStateVector()` to get
+   * `Map<number, number>` (clientID → clock), then compares:
+   *
+   * - Collect the union of all client IDs from both maps.
+   * - For each client, compare clocks (missing client = clock 0).
+   * - If all clocks in `this` ≤ `other` and at least one strictly less → `"behind"`
+   * - If all clocks in `this` ≥ `other` and at least one strictly greater → `"ahead"`
+   * - If all clocks equal → `"equal"`
+   * - Otherwise → `"concurrent"`
+   *
+   * Throws if `other` is not a `YjsVersion`.
+   */
+  compare(other: Version): "behind" | "equal" | "ahead" | "concurrent" {
+    if (!(other instanceof YjsVersion)) {
+      throw new Error(
+        "YjsVersion can only be compared with another YjsVersion",
+      )
+    }
+
+    const thisMap = decodeStateVector(this.sv)
+    const otherMap = decodeStateVector(other.sv)
+
+    // Collect the union of all client IDs
+    const allClients = new Set<number>()
+    for (const id of thisMap.keys()) allClients.add(id)
+    for (const id of otherMap.keys()) allClients.add(id)
+
+    let hasLess = false
+    let hasGreater = false
+
+    for (const clientId of allClients) {
+      const thisClock = thisMap.get(clientId) ?? 0
+      const otherClock = otherMap.get(clientId) ?? 0
+
+      if (thisClock < otherClock) {
+        hasLess = true
+      }
+      if (thisClock > otherClock) {
+        hasGreater = true
+      }
+
+      // Early exit: if we've seen both less and greater, it's concurrent
+      if (hasLess && hasGreater) {
+        return "concurrent"
+      }
+    }
+
+    if (hasLess && !hasGreater) return "behind"
+    if (hasGreater && !hasLess) return "ahead"
+    return "equal"
+  }
+
+  /**
+   * Parse a serialized YjsVersion string back into a YjsVersion.
+   *
+   * The inverse of `serialize()`: base64 → `Uint8Array`.
+   */
+  static parse(serialized: string): YjsVersion {
+    if (serialized === "") {
+      throw new Error("Invalid YjsVersion value: (empty string)")
+    }
+    const bytes = base64ToUint8Array(serialized)
+    return new YjsVersion(bytes)
+  }
+}

package/src/yjs-escape.ts

@@ -0,0 +1,100 @@
+// yjs-escape — Yjs-specific escape hatch for accessing the Y.Doc
+// backing a ref.
+//
+// `yjs(ref)` returns the `Y.Doc` backing a root document ref.
+//
+// The mapping is maintained via a WeakMap from Substrate → Y.Doc,
+// populated by `registerYjsSubstrate()` (called during substrate
+// creation). The `yjs()` function uses `unwrap()` from `@kyneta/schema`
+// to get the substrate, then looks up the Y.Doc.
+//
+// This two-step approach (ref → substrate → Y.Doc) avoids duplicating
+// the ref-tracking WeakMap and composes cleanly with the general
+// `unwrap()` escape hatch.
+//
+// Usage:
+//   import { yjs } from "@kyneta/yjs-schema"
+//
+//   const doc = exchange.get("my-doc", TodoDoc)
+//   const yjsDoc = yjs(doc)  // Y.Doc
+//   yjsDoc.getMap("root").toJSON()  // raw Yjs inspection
+
+import type { Doc as YDoc } from "yjs"
+import type { Substrate } from "@kyneta/schema"
+import { unwrap } from "@kyneta/schema"
+
+// ---------------------------------------------------------------------------
+// Substrate → Y.Doc mapping
+// ---------------------------------------------------------------------------
+
+const substrateToYjsDoc = new WeakMap<Substrate<any>, YDoc>()
+
+// ---------------------------------------------------------------------------
+// registerYjsSubstrate — called during substrate creation
+// ---------------------------------------------------------------------------
+
+/**
+ * Register the Y.Doc backing a Yjs substrate.
+ *
+ * Called by `createYjsSubstrate()` and by `bindYjs`'s factory builder
+ * to enable the `yjs()` escape hatch. Must be called once per substrate
+ * at construction time.
+ */
+export function registerYjsSubstrate(
+  substrate: Substrate<any>,
+  doc: YDoc,
+): void {
+  substrateToYjsDoc.set(substrate, doc)
+}
+
+// ---------------------------------------------------------------------------
+// yjs — Yjs-specific escape hatch
+// ---------------------------------------------------------------------------
+
+/**
+ * Returns the `Y.Doc` backing the given ref.
+ *
+ * This is the Yjs-specific escape hatch for accessing substrate-level
+ * capabilities: raw Yjs API, y-prosemirror/y-codemirror bindings,
+ * undo manager, awareness protocol, Yjs providers (y-websocket,
+ * y-indexeddb, y-webrtc, Hocuspocus, Liveblocks), etc.
+ *
+ * Currently supports root document refs only. Child-level resolution
+ * (e.g. `yjs(doc.title)` → `Y.Text`) is future work.
+ *
+ * @param ref - A root document ref backed by a Yjs substrate
+ * @returns The `Y.Doc` backing the ref
+ * @throws If the ref is not backed by a Yjs substrate
+ *
+ * @example
+ * ```ts
+ * import { yjs } from "@kyneta/yjs-schema"
+ *
+ * const doc = exchange.get("my-doc", TodoDoc)
+ * const yjsDoc = yjs(doc)
+ * console.log(yjsDoc.getMap("root").toJSON())  // raw state
+ * console.log(yjsDoc.clientID)                  // client ID
+ * ```
+ */
+export function yjs(ref: object): YDoc {
+  let substrate: Substrate<any>
+  try {
+    substrate = unwrap(ref)
+  } catch {
+    throw new Error(
+      "yjs() requires a ref backed by a Yjs substrate. " +
+        "Use a doc created by exchange.get() with a bindYjs() schema, " +
+        "or by createYjsDoc().",
+    )
+  }
+
+  const doc = substrateToYjsDoc.get(substrate)
+  if (!doc) {
+    throw new Error(
+      "yjs() requires a ref backed by a Yjs substrate. " +
+        "The ref has a substrate but it is not a Yjs substrate. " +
+        "Use a doc created with a bindYjs() schema or createYjsDoc().",
+    )
+  }
+  return doc
+}