@panphora/clayjs 1.15.1 → 1.16.0

package/src/page/ybind/marks.js

@@ -1,8 +1,20 @@
 import { schema } from "../schema.js";
+import { entriesOfYjs, isOpaque, opaqueMark, yjsFields } from "../format-codec.js";
 
 export const markName = (k) => k.split("!")[0].split("~")[0];
 export const isCancel = (k) => k.includes("!");
 export const marksAttrs = (marks) => { const o = {}; for (const m of marks) o[m.type.name] = m.attrs && Object.keys(m.attrs).length ? { ...m.attrs } : true; return o; };
+// The Yjs fields a set of editor marks stands for. An opaque mark is not a live
+// key of its own: its named descriptors go under their own names and its compact
+// tokens under the reserved one.
+export const marksFields = (marks) => {
+  const o = {};
+  for (const m of marks) {
+    if (isOpaque(m)) { for (const [key, value] of Object.entries(yjsFields(m.attrs.entries))) o[key] = JSON.parse(JSON.stringify(value)); continue; }
+    o[m.type.name] = m.attrs && Object.keys(m.attrs).length ? { ...m.attrs } : true;
+  }
+  return o;
+};
 export const marksKey = (marks) => marks.map((m) => m.type.name + (m.attrs && Object.keys(m.attrs).length ? JSON.stringify(m.attrs) : "")).sort().join("|");
 export function marksFromAttributes(a, blockType) {
   if (blockType === "code_block") return [];
@@ -14,5 +26,17 @@ export function marksFromAttributes(a, blockType) {
     if (v === null || v === undefined || isCancel(k) || cancelled.has(k)) continue;
     best.set(markName(k), v);
   }
-  return [...best].filter(([n]) => schema.marks[n]).map(([n, v]) => schema.marks[n].create(v === true ? null : v));
+  const marks = [];
+  let unknown = null;
+  for (const [n, v] of best) {
+    if (schema.marks[n]) marks.push(schema.marks[n].create(v === true ? null : v));
+    else (unknown ??= {})[n] = v;
+  }
+  // The walk asks this of every text run on every keystroke, and almost none carry a
+  // mark from a newer version, so the codec runs only when one is there.
+  if (unknown) {
+    const opaque = entriesOfYjs(unknown);
+    if (opaque.length) marks.push(opaqueMark(opaque));
+  }
+  return marks;
 }

package/src/page/ybind/migrate.js

@@ -1,10 +1,11 @@
 import * as Y from "../../vendor/yjs.vendor.js";
 import { schema } from "../schema.js";
 import { nodeFromBlock } from "../model.js";
-import { flatten, marksOf } from "../inline.js";
+import { flatten, markKey, marksOf } from "../inline.js";
 import { ATOM, BREAK, DEFAULTS, blocksOfValues, orderedIds, unitsOf, runsOf, canon, hash64, parseModel } from "../projection.js";
 import { blockFromItem } from "../adopt.js";
 import { createPageDoc, attrKeys, defaultOf, SEED } from "./layout.js";
+import { marksFields } from "./marks.js";
 import { blocksFromYValue } from "./decode.js";
 import { PART, enc } from "./provider.js";
 import { boardKey } from "./board.js";
@@ -34,13 +35,54 @@ function sourceBlocks(values) {
   return blocksOfValues(values);
 }
 
+// A mark key as the record keeps it: the codec validates every descriptor, and the
+// known codes and the opaque ones come back in the order the binding writes them.
+const canonicalKey = (key, id) => {
+  let marks;
+  try { marks = marksOf(schema, key); } catch { return fail(`Invalid mark key on block: ${id}`); }
+  return markKey([...marks].sort((a, b) => a.type.rank - b.type.rank));
+};
+
+const markUnits = (block, id) => {
+  const units = unitsOf(block.m, block.c.length);
+  if (!units) fail(`Invalid mark runs on block: ${id}`);
+  for (let i = 0; i < units.length; i++) {
+    const key = units[i];
+    if (typeof key !== "string") fail(`Invalid mark key on block: ${id}`);
+    if (key.startsWith("@") && (key !== BREAK || block.c[i] !== ATOM)) fail(`Unsupported inline atom on block: ${id}`);
+    if (block.t === "code_block" && key) fail(`Unsupported formatting in code block: ${id}`);
+    if (!key.startsWith("@")) units[i] = canonicalKey(key, id);
+  }
+  return units;
+};
+
+// A block type this runtime has no schema for keeps the record it stands for: its
+// own type, its text and marks and every other field a newer peer wrote on it.
+function opaqueRecord(id, block, pos) {
+  const out = { t: block.t, pos };
+  for (const key of Object.keys(block)) {
+    if (key === "t" || key === "pos" || key === "c" || key === "m") continue;
+    out[key] = copy(block[key]);
+  }
+  if (block.c != null && typeof block.c !== "string") fail(`Invalid text on block: ${id}`);
+  out.c = block.c ?? "";
+  if (typeof block.c === "string" && block.m != null) {
+    const units = markUnits(block, id);
+    if (units.some(Boolean)) out.m = runsOf(units);
+  } else if (block.m != null) fail(`Mark runs require flat text on block: ${id}`);
+  return out;
+}
+
 function normalizeBlock(id, block, index) {
-  if (!Object.hasOwn(DEFAULTS, block.t)) fail(`Unsupported hybrid block type: ${block.t}`);
+  const known = Object.hasOwn(DEFAULTS, block.t);
+  if (!known && (typeof block.t !== "string" || !block.t || block.t === "opaque_block")) fail(`Unsupported hybrid block type: ${block.t}`);
+  const pos = String(index + 1).padStart(8, "0");
+  if (!known) return opaqueRecord(id, block, pos);
   const type = schema.nodes[block.t];
   const fields = attrKeys(type);
   const allowed = new Set(["t", "pos", "c", "m", ...fields, ...(block.t === "table" ? ["rows"] : [])]);
-  for (const key of Object.keys(block)) if (!allowed.has(key)) fail(`Unsupported field on block ${id}: ${key}`);
-  const out = { t: block.t, pos: String(index + 1).padStart(8, "0") };
+  const out = { t: block.t, pos };
+  for (const key of Object.keys(block)) if (!allowed.has(key)) out[key] = copy(block[key]);
   for (const key of fields) {
     const value = block[key] === undefined ? defaultOf(type, key) : block[key];
     if (canon(value) !== canon(defaultOf(type, key))) out[key] = copy(value);
@@ -56,14 +98,7 @@ function normalizeBlock(id, block, index) {
   }
   if (block.c != null && typeof block.c !== "string" && !Array.isArray(block.c)) fail(`Invalid text on block: ${id}`);
   if (typeof block.c === "string" && block.m != null) {
-    const units = unitsOf(block.m, block.c.length);
-    if (!units) fail(`Invalid mark runs on block: ${id}`);
-    for (let i = 0; i < units.length; i++) {
-      const key = units[i];
-      if (typeof key !== "string") fail(`Invalid mark key on block: ${id}`);
-      if (key.startsWith("@") && (key !== BREAK || block.c[i] !== ATOM)) fail(`Unsupported inline atom on block: ${id}`);
-      if (block.t === "code_block" && key) fail(`Unsupported formatting in code block: ${id}`);
-    }
+    markUnits(block, id);
   } else if (block.m != null) fail(`Mark runs require flat text on block: ${id}`);
   const node = nodeFromBlock(schema, id, block);
   node.check();
@@ -107,7 +142,7 @@ export function buildPageValue(values, clientID) {
           if (key === BREAK) {
             for (let i = 0; i < length; i++) pd.flow.insertEmbed(pd.flow.length, { br: 1 }, {});
           } else {
-            const attrs = Object.fromEntries(marksOf(schema, key).map((mark) => [`${mark.type.name}~${clientID}-m${(op++).toString(36)}`, Object.keys(mark.attrs).length ? copy(mark.attrs) : true]));
+            const attrs = Object.fromEntries(Object.entries(marksFields(marksOf(schema, key))).map(([name, value]) => [`${name}~${clientID}-m${(op++).toString(36)}`, value && typeof value === "object" ? copy(value) : value]));
             pd.flow.insert(pd.flow.length, c.slice(offset, offset + length), attrs);
           }
           offset += length;

package/src/page/ybind/moves.js

@@ -59,6 +59,7 @@ const MAX_INDENT = 8;
 const ATTR_RULES = {
   indent: (value, block) => block.t === "list_item" && Number.isInteger(value) && value >= 0 && value <= MAX_INDENT,
   open: (value, block) => block.t === "toggle" && typeof value === "boolean",
+  parent: (value, block, byId) => value === null || (typeof value === "string" && byId.get(value)?.t === "toggle" && !inSubtree(byId, value, block.id)),
 };
 
 const attrChanges = (args, byId) => {
@@ -69,7 +70,7 @@ const attrChanges = (args, byId) => {
     const block = byId.get(id);
     if (!block || !fields || typeof fields !== "object" || Array.isArray(fields)) return null;
     for (const [field, value] of Object.entries(fields)) {
-      if (!ATTR_RULES[field] || !ATTR_RULES[field](value, block)) return null;
+      if (!ATTR_RULES[field] || !ATTR_RULES[field](value, block, byId)) return null;
       out.push({ id, field, value });
     }
   }
@@ -118,6 +119,9 @@ export function moveBlocks({ pd, view, um, origins, render }, args = {}) {
   const hasParent = args?.parent !== undefined;
 
   const current = walk(pd, origins);
+  // An opaque block is an operation this runtime must refuse before writing: it
+  // has no turn-into, it cannot be carried, and its record is never rewritten.
+  const protectedIds = new Set(current.visible.filter((seg) => seg.opaque).map((seg) => seg.id));
   const list = visibleBlocks(pd, current, view);
   const byId = new Map(list.map((block) => [block.id, block]));
   const order = list.map((block) => block.id);
@@ -126,10 +130,12 @@ export function moveBlocks({ pd, view, um, origins, render }, args = {}) {
   for (const id of ids) if (!byId.has(id)) return refuse("missing-source");
   const changes = attrChanges(args, byId);
   if (!changes) return refuse("invalid-attrs");
+  const parentChanges = changes.filter((change) => change.field === "parent");
 
   const roots = new Set(ids.filter((id) => !ids.some((other) => other !== id && isAncestor(byId, other, id))));
   const selected = list.filter((block) => ids.some((root) => inSubtree(byId, block.id, root)));
   const selectedSet = new Set(selected.map((block) => block.id));
+  if (selected.some((block) => protectedIds.has(block.id)) || changes.some((change) => protectedIds.has(change.id))) return refuse("unsupported-format");
 
   const segments = new Map(current.visible.map((seg) => [seg.id, seg]));
   for (const block of selected) {
@@ -174,14 +180,33 @@ export function moveBlocks({ pd, view, um, origins, render }, args = {}) {
   const movedIds = selected.map((block) => block.id);
   if (sameOrder && !movedRoots.length && !changes.length) return { status: "unchanged" };
 
+  const caret = (() => {
+    const sel = view?.state.selection;
+    if (!(sel instanceof pmState.TextSelection)) return null;
+    const at = (pos) => { const $pos = view.state.doc.resolve(pos); return $pos.depth ? { id: $pos.node(1).attrs.id, offset: $pos.parentOffset } : null; };
+    const anchor = at(sel.anchor), head = at(sel.head);
+    return anchor && head && (selectedSet.has(anchor.id) || selectedSet.has(head.id)) ? { anchor, head } : null;
+  })();
+  const restoreCaret = () => {
+    if (!caret || !view) return;
+    const place = ({ id, offset }) => {
+      let found = null;
+      view.state.doc.forEach((node, pos) => { if (found === null && node.attrs.id === id) found = pos + 1 + Math.min(offset, node.content.size); });
+      return found;
+    };
+    const anchor = place(caret.anchor), head = place(caret.head);
+    if (anchor !== null && head !== null) view.dispatch(view.state.tr.setSelection(pmState.TextSelection.create(view.state.doc, anchor, head)).setMeta(REMOTE_META, true).setMeta("addToHistory", false));
+  };
   if (sameOrder) {
     um?.stopCapturing();
     pd.ydoc.transact(() => {
-      for (const block of movedRoots) setParent(pd, segments.get(block.id), block.id, parent);
-      for (const change of changes) setAttr(pd, segments.get(change.id), change.id, change.field, change.value);
+      if (parentChanges.length) for (const change of parentChanges) setParent(pd, segments.get(change.id), change.id, change.value);
+      else for (const block of movedRoots) setParent(pd, segments.get(block.id), block.id, parent);
+      for (const change of changes) if (change.field !== "parent") setAttr(pd, segments.get(change.id), change.id, change.field, change.value);
     }, LOCAL);
     um?.stopCapturing();
     render();
+    restoreCaret();
     return { status: "moved", ids: movedIds };
   }
 
@@ -216,13 +241,6 @@ export function moveBlocks({ pd, view, um, origins, render }, args = {}) {
     const baseline = intents.restores.find((restore) => restore.key === winner?.key)?.baseline ?? { type: seg.t, attrs: seg.fields ?? {} };
     promotions.set(seg.sourceKey, { covers: removalsOf.map((removal) => removal.key), baseline: JSON.parse(JSON.stringify(baseline)) });
   }
-  const caret = (() => {
-    const sel = view?.state.selection;
-    if (!(sel instanceof pmState.TextSelection)) return null;
-    const at = (pos) => { const $pos = view.state.doc.resolve(pos); return $pos.depth ? { id: $pos.node(1).attrs.id, offset: $pos.parentOffset } : null; };
-    const anchor = at(sel.anchor), head = at(sel.head);
-    return anchor && head && (selectedSet.has(anchor.id) || selectedSet.has(head.id)) ? { anchor, head } : null;
-  })();
   if (!removals.length) throw new Error("The selected blocks have no live physical markers to retire");
 
   um?.stopCapturing();
@@ -270,19 +288,12 @@ export function moveBlocks({ pd, view, um, origins, render }, args = {}) {
       pd.gone.set(copyActiveKey(physicalKeyOf(dest[0])), true);
     }
     if (spans.length) origins.recordSpans(spans);
-    for (const block of movedRoots) setParent(pd, segments.get(block.id), block.id, parent);
-    for (const change of changes) setAttr(pd, segments.get(change.id), change.id, change.field, change.value);
+    if (parentChanges.length) for (const change of parentChanges) setParent(pd, segments.get(change.id), change.id, change.value);
+    else for (const block of movedRoots) setParent(pd, segments.get(block.id), block.id, parent);
+    for (const change of changes) if (change.field !== "parent") setAttr(pd, segments.get(change.id), change.id, change.field, change.value);
   }, LOCAL);
   um?.stopCapturing();
   render();
-  if (caret && view) {
-    const place = ({ id, offset }) => {
-      let found = null;
-      view.state.doc.forEach((node, pos) => { if (found === null && node.attrs.id === id) found = pos + 1 + Math.min(offset, node.content.size); });
-      return found;
-    };
-    const anchor = place(caret.anchor), head = place(caret.head);
-    if (anchor !== null && head !== null) view.dispatch(view.state.tr.setSelection(pmState.TextSelection.create(view.state.doc, anchor, head)).setMeta(REMOTE_META, true).setMeta("addToHistory", false));
-  }
+  restoreCaret();
   return { status: "moved", ids: movedIds };
 }

package/src/page/ybind/positions.js

@@ -0,0 +1,50 @@
+// Where a position sits on the page: the map between a ProseMirror position in
+// the projected document and an index in the page's Yjs `flow`. Extracted from
+// bridge.js so the same functions keep this tab's own caret through a peer's edit
+// and place a peer's caret here, instead of each approximating the other.
+import { tokenIndex, yBefore } from "./tokens.js";
+
+export const yIndexAt = (doc, holes, pos, visible) => yBefore(holes, tokenIndex(doc, new Map(), visible)[pos]);
+
+export function posAtY(doc, holes, tokens, y, visible = null) {
+  let lo = 0, hi = tokens;
+  while (lo < hi) { const mid = (lo + hi + 1) >> 1; if (yBefore(holes, mid) <= y) lo = mid; else hi = mid - 1; }
+  const idx = tokenIndex(doc, new Map(), visible);
+  let first = -1;
+  for (let p = 0; p < idx.length; p++) {
+    if (idx[p] < lo) continue;
+    if (idx[p] > lo) break;
+    if (first < 0) first = p;
+    if (doc.resolve(p).parent.inlineContent) return p;
+  }
+  return first < 0 ? doc.content.size : first;
+}
+
+export const cellPoint = (doc, pos) => {
+  const $pos = doc.resolve(pos);
+  if ($pos.depth !== 3 || $pos.parent.type.name !== "table_cell") return null;
+  return { table: $pos.node(1).attrs.id, row: $pos.node(2).attrs.id, col: $pos.parent.attrs.col, offset: $pos.parentOffset };
+};
+
+// The cell a point names, as its own range: what an outline draws and what a
+// character offset would pretend to know.
+export const cellRange = (doc, point) => {
+  let found = null;
+  doc.forEach((table, at) => {
+    if (found !== null || table.type.name !== "table" || table.attrs.id !== point.table) return;
+    table.forEach((row, rowAt) => {
+      if (found !== null || row.attrs.id !== point.row) return;
+      row.forEach((cell, cellAt) => {
+        if (found === null && cell.attrs.col === point.col) found = { from: at + 2 + rowAt + cellAt, to: at + 2 + rowAt + cellAt + cell.nodeSize, block: point.table, row: point.row, col: point.col };
+      });
+    });
+  });
+  return found;
+};
+
+export const cellAt = (doc, point) => {
+  const range = cellRange(doc, point);
+  if (!range) return null;
+  const cell = doc.nodeAt(range.from);
+  return range.from + 1 + Math.min(point.offset ?? 0, cell ? cell.content.size : 0);
+};

package/src/page/ybind/provider.js

@@ -3,6 +3,7 @@ import { toBase64, fromBase64 } from "../../vendor/yjs.vendor.js";
 import { NET } from "./layout.js";
 import { ORIGINS_ORIGIN } from "./origins.js";
 import { PART_KEY, BASE64, entriesOf, captureYValue } from "./parts.js";
+import { containsUpdate } from "./recovery-proof.js";
 
 export { entriesOf } from "./parts.js";
 
@@ -11,7 +12,6 @@ export const dec = (s) => fromBase64(s);
 
 export const PART = 12 * 1024;
 const baseOf = (k) => PART_KEY.exec(k)?.[1] ?? k;
-const sameBytes = (a, b) => a.length === b.length && a.every((v, i) => v === b[i]);
 
 export function yValueOf(ydoc) {
   return { v: 1, g: 0, y: { seed: enc(Y.encodeStateAsUpdateV2(ydoc)) } };
@@ -75,6 +75,8 @@ export function docFromValue(value, { strict = false } = {}) {
   }
 }
 
+const encoder = typeof TextEncoder === "function" ? new TextEncoder() : null;
+const utf8Bytes = (text) => (encoder ? encoder.encode(text).length : text.length * 2);
 const sameDeletes = (a, b) => Y.equalDeleteSets(Y.createDeleteSetFromStructStore(a.store), Y.createDeleteSetFromStructStore(b.store));
 const svEqual = (a, b) => { if (a.size !== b.size) return false; for (const [c, n] of a) if (b.get(c) !== n) return false; return true; };
 
@@ -95,6 +97,9 @@ export function createYProvider({ store, ydoc, compact: limits = {}, setTimer =
   const me = ydoc.clientID.toString(36);
   const nextKey = (p) => `${p}${me}-${(counter++).toString(36)}`;
   let timer = null;
+  const generation = store.value.g;
+  // Keys this tab wrote itself: the updates and offers the room has not confirmed yet.
+  const own = new Set();
 
   const partsOf = (key, s) => {
     if (s.length <= PART) return [[key, s]];
@@ -104,6 +109,7 @@ export function createYProvider({ store, ydoc, compact: limits = {}, setTimer =
   const put = (key, bytes) => {
     canCompact = true;
     known.add(key);
+    own.add(key);
     const parts = partsOf(key, enc(bytes));
     store.handle.update((data) => { for (const [k, v] of parts) data.y[k] = v; });
   };
@@ -163,20 +169,70 @@ export function createYProvider({ store, ydoc, compact: limits = {}, setTimer =
   const compact = { keys: 32, room: 3840 * 1024, ...limits };
   let fold = null;
   let quietUntil = 0;
+  const COVER_MAX = 256;
   const coverCache = new Map();
   const drop = (keys) => store.handle.update((data) => { for (const k of keys) delete data.y[k]; });
   const keysOf = (auth, bases) => Object.keys(auth).filter((k) => bases.has(baseOf(k)));
+  // A key is retired only while what this tab shows under it is still what the proof was
+  // made over: a late write under the same key keeps its own entry.
+  const retire = (auth, keys) => { const visible = store.value.y ?? {}; drop(keys.filter((k) => k in visible && visible[k] === auth[k])); };
 
+  // `known` and the coverage memo keep live references only: the authoritative entries,
+  // this tab's own unconfirmed writes and the fold in progress. Pruned after a rebase, a
+  // fold and a refusal. A key that is forgotten can be decoded again; Yjs is idempotent.
+  function prune() {
+    const auth = store.authoritative.y ?? {};
+    const visible = store.value.y ?? {};
+    for (const k of own) if (!(k in visible)) own.delete(k);
+    const held = fold ? [...fold.parts.map(([k]) => k), ...fold.inputs.keys()] : [];
+    const retained = new Set([...Object.keys(auth), ...own, ...held]);
+    for (const key of known) if (!retained.has(key)) known.delete(key);
+    const values = new Map(entriesOf(auth));
+    for (const [key, proof] of coverCache) if (!retained.has(key) || values.get(key) !== proof.value) coverCache.delete(key);
+  }
+
+  // Whether a fold's merged bytes already carry this input: merging it in again changes
+  // neither an insertion nor a deletion. Equal state vectors say nothing about deletes.
+  function coveredBy(key, value, foldKey, base) {
+    let proof = coverCache.get(key);
+    if (proof && proof.value !== value) { coverCache.delete(key); proof = null; }
+    if (!proof) {
+      if (coverCache.size >= COVER_MAX) coverCache.delete(coverCache.keys().next().value);
+      proof = { value, folds: new Map() };
+      coverCache.set(key, proof);
+    }
+    if (!proof.folds.has(foldKey)) proof.folds.set(foldKey, containsUpdate(base, dec(value)));
+    return proof.folds.get(foldKey);
+  }
+
+  // An input the authority no longer has was retired already; one it holds differently
+  // was rewritten after the fold was built, and the fold no longer speaks for it.
+  const stillHeld = (auth, active) => { for (const [k, v] of active.held) if (auth[k] !== undefined && auth[k] !== v) return false; return true; };
+  const coversInputs = (active) => {
+    const base = dec(active.merged);
+    for (const [k, v] of active.inputs) if (v !== null && !coveredBy(k, v, active.key, base)) return false;
+    return true;
+  };
+
+  // Inputs retire only once the authority holds every proposed part with the identical
+  // value, still holds each input as it was captured, and the fold covers every one of
+  // them. Anything less keeps the originals; pending confirmation keeps them too.
   function finishFold(auth) {
-    if (fold.parts.every((k) => k in auth)) {
-      drop(keysOf(auth, fold.inputs));
+    const active = fold;
+    const whole = captureYValue({ v: 1, y: auth }).y;
+    const arrived = active.parts.filter(([k, v]) => auth[k] === v && whole[k] === v);
+    if (arrived.length !== active.parts.length) {
+      if (store.pending.length) return false;
       fold = null;
+      if (arrived.length) retire(auth, arrived.map(([k]) => k));
+      quietUntil = entriesOf(auth).length + compact.keys;
+      prune();
       return true;
     }
-    if (store.pending.length) return false;
-    drop(fold.parts.filter((k) => k in auth));
-    quietUntil = entriesOf(auth).length + compact.keys;
+    if (store.pending.length || !stillHeld(auth, active) || !coversInputs(active)) return false;
     fold = null;
+    retire(auth, keysOf(auth, active.inputs));
+    prune();
     return true;
   }
 
@@ -185,11 +241,7 @@ export function createYProvider({ store, ydoc, compact: limits = {}, setTimer =
     const val = new Map(entries);
     const norm = new Map(folds);
     const normal = (k) => { const v = norm.get(k); if (typeof v === "string") norm.set(k, Y.mergeUpdatesV2([dec(v)])); return norm.get(k); };
-    const covered = (k, v, mk) => {
-      const c = `${k}|${mk}`;
-      if (!coverCache.has(c)) { const b = normal(mk); coverCache.set(c, sameBytes(Y.mergeUpdatesV2([b, dec(v)]), b)); }
-      return coverCache.get(c);
-    };
+    const covered = (k, v, mk) => coveredBy(k, v, mk, normal(mk));
     const gone = new Set();
     for (const [k, v] of entries) {
       if (!crowded && k[0] !== "m") continue;
@@ -201,31 +253,36 @@ export function createYProvider({ store, ydoc, compact: limits = {}, setTimer =
       }
     }
     if (!gone.size) return false;
-    drop(keysOf(auth, gone));
+    retire(auth, keysOf(auth, gone));
     return true;
   }
 
   function maybeCompact() {
-    if (!canCompact) return false;
+    if (stopped || !canCompact || store.value.g !== generation) return false;
     const auth = store.authoritative.y;
     if (fold) return finishFold(auth);
     const entries = entriesOf(auth);
     const crowded = entries.length > Math.max(compact.keys, quietUntil);
     if (!crowded && entries.filter(([k]) => k[0] === "m").length < 2) return false;
-    if (dropCovered(auth, entries, crowded)) return true;
+    if (dropCovered(auth, entries, crowded)) { prune(); return true; }
     if (!crowded) return false;
     const merged = enc(Y.mergeUpdatesV2(entries.map(([, v]) => dec(v))));
-    const used = Object.values(auth).reduce((n, v) => n + v.length, 0);
-    if (used + merged.length > compact.room) {
+    const key = nextKey("m");
+    const parts = partsOf(key, merged);
+    const candidate = { ...store.value, y: { ...store.value.y } };
+    for (const [k, v] of parts) candidate.y[k] = v;
+    if (utf8Bytes(JSON.stringify(candidate)) > compact.room) {
       quietUntil = entries.length + compact.keys;
       return false;
     }
-    const key = nextKey("m");
-    known.add(key);
-    const parts = partsOf(key, merged);
     const captured = captureYValue({ v: 1, y: auth }).y;
     const incompleteFolds = Object.keys(auth).filter((k) => !(k in captured)).map(baseOf).filter((k) => k[0] === "m");
-    fold = { key, parts: parts.map(([k]) => k), inputs: new Set([...entries.map(([k]) => k), ...incompleteFolds]) };
+    const inputs = new Map(entries);
+    for (const k of incompleteFolds) inputs.set(k, null);
+    const held = new Map();
+    for (const k of keysOf(auth, inputs)) if (inputs.get(baseOf(k)) !== null) held.set(k, captured[k] ?? auth[k]);
+    fold = { key, parts, merged, inputs, held };
+    known.add(key);
     store.handle.update((data) => { for (const [k, v] of parts) data.y[k] = v; });
     stats.compactions++;
     return true;
@@ -264,6 +321,7 @@ export function createYProvider({ store, ydoc, compact: limits = {}, setTimer =
     } else {
       for (const [k, v] of entriesOf(ev.next.y ?? {})) applyKey(k, v);
     }
+    prune();
     schedule(ev.source !== "remote" || !ops, ev.source !== "remote");
   });
 
@@ -272,14 +330,18 @@ export function createYProvider({ store, ydoc, compact: limits = {}, setTimer =
   // the refused one, so they are all taken back, with the parts of updates the room does not
   // hold whole, and the room's page stays one a checkpoint can render.
   const offDelivery = store.onDelivery((event) => {
-    if (event.phase !== "refused" || event.code !== "too-large" || stopped) return;
+    if (event.phase !== "refused") return;
+    if (event.code !== "too-large" || stopped) { prune(); return; }
     stop("too-large");
     const auth = store.authoritative.y ?? {};
     const whole = captureYValue({ v: 1, y: auth }).y;
     const unconfirmed = Object.keys(store.value.y ?? {}).filter((k) => !(k in auth));
     const partial = Object.keys(auth).filter((k) => PART_KEY.test(k) && !(k in whole) && baseOf(k).slice(1).startsWith(`${me}-`));
     if (unconfirmed.length || partial.length) drop([...unconfirmed, ...partial]);
+    prune();
   });
 
-  return { known, stats, settle, reconcile, get reconcilePending() { return reconcilePending; }, get stopped() { return stopped; }, compact: maybeCompact, dispose() { off(); offDelivery(); ydoc.off("updateV2", onUpdate); if (timer) clearTimer(timer); } };
+  return { known, stats, settle, reconcile, get reconcilePending() { return reconcilePending; }, get stopped() { return stopped; },
+    // Test-only snapshot of the counters and cache sizes; no mutable set leaves the provider.
+    _diagnostics: () => ({ known: known.size, cover: coverCache.size, fold: fold ? fold.parts.length : 0, generation }), compact: maybeCompact, dispose() { off(); offDelivery(); ydoc.off("updateV2", onUpdate); if (timer) clearTimer(timer); } };
 }