@panphora/clayjs 1.17.0 → 1.18.0

package/src/lib/unsaved-state.js

@@ -52,9 +52,9 @@ export function hasUnsavedState() {
 // Ask every pending source that can finish on its own (a checkpoint, a queued
 // upload) to do so, and wait. Sources without a flush (an open composition)
 // stay pending; they are listed in the answer, never waited on.
-export async function flushUnsaved() {
+export async function flushUnsaved(context = {}) {
   const flushes = [...sources.values()].filter((s) => typeof s.flush === 'function' && isPending(s));
-  const results = await Promise.all(flushes.map((s) => Promise.resolve().then(s.flush).catch(() => ({ ok: false }))));
+  const results = await Promise.all(flushes.map((s) => Promise.resolve().then(() => s.flush(context)).catch(() => ({ ok: false }))));
   return { ok: results.every((r) => r?.ok !== false), pending: pendingIds() };
 }
 

package/src/live/fast-lane.js

@@ -231,7 +231,7 @@ export function createFastLane({
     lastSentAt = now();
     let result;
     try {
-      result = await transport.send(clientId, { name, n: frame.n, ops: frame.ops, epoch: sentEpoch, ...(conn !== undefined && { conn }), ...(frame.receiptGeneration !== null && { receiptGeneration: frame.receiptGeneration }), ...(frame.pageGeneration !== null && PAGE_STORE.test(name) && { pageGeneration: frame.pageGeneration }), ...(frame.adopted && { adopted: frame.adopted }) });
+      result = await transport.send(clientId, { name, n: frame.n, ops: frame.ops, epoch: sentEpoch, ...(conn !== undefined && { conn }), ...(frame.receiptGeneration !== null && { receiptGeneration: frame.receiptGeneration }), ...(frame.pageGeneration !== null && PAGE_STORE.test(name) && { pageGeneration: frame.pageGeneration }), ...(frame.adopted && { adopted: frame.adopted }), ...(PAGE_STORE.test(name) && lane.compaction?.v === 1 && { compaction: 1 }) });
     } catch {
       result = null;
     }
@@ -386,6 +386,12 @@ export function createFastLane({
 
   function apply(frame) {
     seq = frame.seq;
+    if (frame.from === "compactor") {
+      holdUntil = 0;
+      if (timer !== null) clearTimer(timer);
+      timer = null;
+      schedule();
+    }
     const store = entries.get(frame.name)?.store;
     if (!store || abandoned.has(frame.name)) return;
     if (!roomNames.has(frame.name)) {
@@ -510,6 +516,7 @@ export function createFastLane({
       prune(name, lane);
       const held = lane.room;
       lane.room = !!room;
+      lane.compaction = room?.compaction ?? null;
       if (!room) {
         // Not in the room (too large, too many stores, or opened after it):
         // there are no frames of it to miss. Already out: its value still
@@ -569,7 +576,7 @@ export function createFastLane({
           frame.pageGeneration = pageGeneration;
         }
         lane.awaiting = resend;
-        entry.store.rebase(room.value, { source: "state", dropIds, room: { seq, fresh: true, moves: state.moves?.[name] ?? null, floor: Number.isInteger(room.floor) ? room.floor : null } });
+        entry.store.rebase(room.value, { source: "state", dropIds, room: { seq, fresh: true, moves: state.moves?.[name] ?? null, floor: Number.isInteger(room.floor) ? room.floor : null, compaction: lane.compaction } });
         lane.unsent = unframed(name, lane);
       } else {
         const seen = room.seen ? room.seen[clientId] : undefined;
@@ -599,7 +606,7 @@ export function createFastLane({
           }
         }
         lane.awaiting = next;
-        entry.store.rebase(room.value, { source: "state", dropIds, room: { seq, fresh: false, moves: state.moves?.[name] ?? null, floor: Number.isInteger(room.floor) ? room.floor : null } });
+        entry.store.rebase(room.value, { source: "state", dropIds, room: { seq, fresh: false, moves: state.moves?.[name] ?? null, floor: Number.isInteger(room.floor) ? room.floor : null, compaction: lane.compaction } });
         // Back in the room after a spell at save speed: frame what was made meanwhile.
         if (rejoined) {
           lane.unsent = unframed(name, lane);

package/src/live/local-lane.js

@@ -17,13 +17,25 @@
 // stamp (or, without one, the last version known to be in the file) picks
 // where in the log it starts, the moves after that go back on top, and the
 // merge itself is logged as a diff so a later file stamped before it keeps it.
+//
+// In the room scope the page store is a host-mode room, compacted by the tab that
+// sequences it: an update is one whole client key, a refusal answers with the
+// host's code, and a due room folds its keys in an idle task (a tail merge or a
+// full, each proven before it is written) and sends the commit as a move of its
+// own. A save merges as a union of keys, under names this room makes. The scope is
+// part of the lane key, so a tab of the lane before it never shares a sequencer.
 
 import { localChannel } from "./local-channel.js";
 import { applyOp, applyTracked, checkOp, cloneJson, undoTracked, validateOp, serializeLive, textHash, diffOps } from "./ops.js";
 import { parseJsonRelaxed } from "../vendor/hyper-morph-json.vendor.js";
 import { movesSince } from "./moves.js";
+import { PART_KEY, entriesOf } from "../page/ybind/parts.js";
+import { COMPACTION, ROOM_ERROR_STATUS, admitPageOps, planCompaction, buildCandidate, checkUpdate, pageShape, createRangeCache, unionFileEntries } from "./page-compaction.js";
 
-const VERSION = 4;
+// The lane's own message version, carried by every message it posts: a tab of
+// another version drops its frames even if a scope were ever shared. The shared
+// channel keeps its wire version, which presence rides beside.
+const VERSION = 5;
 export const REQUEST_MS = 2000;
 export const CLOSE_WAIT_MS = 500;
 export const LOG_ENTRIES = 1024;
@@ -58,11 +70,22 @@ const encoder = new TextEncoder();
 // host floor. The message protocol inside the scope is unchanged.
 export const FORMAT_SCOPE = "page-format:1";
 
+// The room scope: this lane's sequencer is a host-mode room, so a tab of the lane
+// before it, which cannot read that room, is never on the same channel or lock.
+export const ROOM_SCOPE = "page-rooms:2";
+
 export function localLaneKey(floor = 0, path = window.location.pathname) {
   const base = "clay-live:" + path;
-  return floor >= 1 ? `${base}:${FORMAT_SCOPE}` : base;
+  return `${floor >= 1 ? `${base}:${FORMAT_SCOPE}` : base}:${ROOM_SCOPE}`;
+}
+
+function idle(fn) {
+  if (typeof requestIdleCallback === "function") requestIdleCallback(fn, { timeout: 1000 });
+  else setTimeout(fn, 0);
 }
 
+const isYValue = (value) => !!value && typeof value === "object" && typeof value.y === "object" && value.y !== null && !Array.isArray(value.y);
+
 export function localLaneSupported() {
   return typeof BroadcastChannel === "function" && typeof navigator !== "undefined" && typeof navigator.locks?.request === "function";
 }
@@ -79,12 +102,16 @@ function randomId() {
  * @param {string} [options.key]
  * @param {object} [options.channel] - the document's channel, when a presence
  *   service in this tab already owns one: the two share it rather than open a second
+ * @param {boolean} [options.rooms] - the page's room scope: the channel and the
+ *   lock of the host-mode lane, which a tab of the lane before it never joins
  */
-// `key` is the room's channel without a floor suffix; `floor` scopes it.
-export function localTransport({ seed, key = "clay-live:" + window.location.pathname, channel: shared = null, floor = 0 }) {
+// `key` is the room's channel without a floor suffix; `floor` scopes it, and
+// `rooms` puts it in the room scope.
+export function localTransport({ seed, key = "clay-live:" + window.location.pathname, channel: shared = null, floor = 0, rooms = false }) {
   const clientId = randomId();
   let scopeFloor = floor >= 1 ? 1 : 0;
-  let laneKey = scopeFloor ? `${key}:${FORMAT_SCOPE}` : key;
+  const scopeOf = (f) => `${key}${f >= 1 ? `:${FORMAT_SCOPE}` : ""}${rooms ? `:${ROOM_SCOPE}` : ""}`;
+  let laneKey = scopeOf(scopeFloor);
   let channel = null;
   let unsubscribe = null;
   let handlers = null;
@@ -118,7 +145,7 @@ export function localTransport({ seed, key = "clay-live:" + window.location.path
   let mirror = { epoch: null, seq: 0, sequencer: null, closedBy: null, stale: false, seen: new Map(), stores: new Map() };
 
   function post(message) {
-    channel?.post(message);
+    channel?.post({ ...message, version: VERSION });
   }
 
   // Over either cap the oldest entries go, and a file older than them can no
@@ -166,6 +193,7 @@ export function localTransport({ seed, key = "clay-live:" + window.location.path
     const meta = {};
     for (const [name, m] of mirror.stores) {
       stores[name] = { value: cloneJson(m.value), ephemeral: m.ephemeral, seen: receiptsFor(name), last: m.last, landed: landedIn(m) };
+      if (name === "page" && isYValue(m.value)) stores[name].compaction = { v: COMPACTION.version, loose: COMPACTION.loose, activeBytes: COMPACTION.activeBytes };
       meta[name] = { log: m.log, last: m.last, landed: m.landed, floor: m.floor, behind: m.behind, file: m.file, inFile: m.inFile };
     }
     const state = { epoch: room.epoch, seq: room.seq, writer: clientId, stores, mirror: { seen: [...mirror.seen], stores: meta } };
@@ -272,7 +300,7 @@ export function localTransport({ seed, key = "clay-live:" + window.location.path
     mirror.closedBy = null;
     const lastN = new Map();
     for (const [nKey, n] of mirror.seen) lastN.set(nKey, { n, seq: 0 });
-    room = { epoch: mirror.epoch, seq: mirror.seq, lastN };
+    room = { epoch: mirror.epoch, seq: mirror.seq, lastN, host: pageHost() };
     for (const [name, frame] of kept) {
       const m = mirror.stores.get(name);
       if (!m || m.ephemeral) continue;
@@ -351,6 +379,111 @@ export function localTransport({ seed, key = "clay-live:" + window.location.path
       });
   }
 
+  // A room's host mode: the sources it retired, the last full fold and the
+  // deletions since it (which tell a room that is out of room from one that can
+  // still be compacted), a build failure that stops it, the range cache the union
+  // merge reads, and the idle task the room has asked for.
+  const pageHost = () => ({ retired: new Set(), lastFull: null, deletesSince: 0, failed: null, ranges: createRangeCache(), task: null, overflow: false });
+
+  const pageStore = () => {
+    const m = mirror.stores.get("page");
+    return m && !m.ephemeral && isYValue(m.value) ? m : null;
+  };
+
+  const pageBytes = (value) => encoder.encode(serializeLive(value)).length;
+
+  // A host-mode refusal answers as the server's does: the status its code has and
+  // the code itself, which is what a tab acts on.
+  const refused = (error, retryAfter = 0) => ({ status: ROOM_ERROR_STATUS[error] ?? 409, body: { code: error }, retryAfter });
+
+  // The room is due a compaction: one idle task, in this tab. The plan, the build
+  // and the commit all run inside it without yielding, so no job outlives the
+  // sequencer that made it and a hand-off cannot carry one to its successor.
+  function due(overflow = false) {
+    if (!room) return;
+    const host = room.host;
+    host.overflow ||= overflow;
+    if (host.task) return;
+    const target = room;
+    host.task = idle(() => {
+      host.task = null;
+      const over = host.overflow;
+      host.overflow = false;
+      if (stopped || room !== target || host.failed) return;
+      compact(host, { overflow: over });
+    });
+  }
+
+  function compact(host, { overflow }) {
+    const page = pageStore();
+    if (!page) return;
+    const plan = planCompaction(page.value.y, { overflow }, COMPACTION);
+    if (!plan) return;
+    const built = buildCandidate({ kind: plan.kind, sources: plan.sources });
+    if (typeof built.update !== "string") {
+      host.failed = { reason: built.failure };
+      return;
+    }
+    commitPage(host, page, plan, built.update);
+  }
+
+  // Every source still there with the identical string, and the candidate proven:
+  // then the room retires the sources for one checkpoint key of its own, named for
+  // this epoch and position, and the commit goes out as a move like any other.
+  function commitPage(host, m, plan, update) {
+    const y = m.value.y;
+    const sources = new Map(plan.sources);
+    const physical = Object.keys(y).filter((k) => {
+      const part = PART_KEY.exec(k);
+      return sources.has(part ? part[1] : k) || (part && plan.kind === "full");
+    });
+    const held = new Map(entriesOf(Object.fromEntries(physical.map((k) => [k, y[k]]))));
+    for (const [key, value] of sources) if (held.get(key) !== value) return;
+    if (!checkUpdate(update)) return;
+    const key = `${plan.kind === "full" ? "c" : "t"}${textHash(room.epoch).slice(0, 8)}-${(room.seq + 1).toString(36)}`;
+    if (Object.hasOwn(y, key)) return;
+    const ops = [...physical.map((source) => validateOp({ op: "del", path: ["y", source] })), validateOp({ op: "set", path: ["y", key], value: update })];
+    const before = pageBytes(m.value);
+    let trial;
+    try {
+      trial = applyAll(m.value, ops);
+    } catch {
+      return;
+    }
+    const bytes = pageBytes(trial.value);
+    trial.undo();
+    if (bytes > COMPACTION.activeBytes && bytes > before) {
+      // A full fold of the whole page that still does not fit: the page is as small as it
+      // gets, so the next write is refused as full instead of waiting for another fold.
+      if (plan.kind === "full" && physical.length === Object.keys(y).length) {
+        host.lastFull = { bytes: before, seq: room.seq };
+        host.deletesSince = 0;
+      }
+      return;
+    }
+    const clean = !m.behind && m.landed === m.last;
+    for (const [source] of plan.sources) {
+      host.retired.add(source);
+      if (host.retired.size > COMPACTION.retired) host.retired.delete(host.retired.values().next().value);
+    }
+    room.seq += 1;
+    if (plan.kind === "full") {
+      host.lastFull = { bytes, seq: room.seq };
+      host.deletesSince = 0;
+    }
+    const data = { epoch: room.epoch, seq: room.seq, name: "page", ops, from: "compactor", n: 0, kind: plan.kind };
+    advance(data);
+    if (clean) m.landed = room.seq;
+    post({ type: "data", data });
+    handlers.frame(cloneJson(data));
+    if (clean) {
+      tellLanded(["page"]);
+      post({ type: "landed", name: "page", at: { epoch: room.epoch, seq: room.seq } });
+    }
+    const shape = pageShape(m.value.y, COMPACTION);
+    if (shape.loose >= COMPACTION.soft || !shape.normal) due();
+  }
+
   function roomApply(from, frame) {
     if (!frame || typeof frame.name !== "string" || !Number.isInteger(frame.n) || frame.n < 1 || !Array.isArray(frame.ops) || !frame.ops.length) {
       return { status: 400, body: { error: "bad-frame" }, retryAfter: 0 };
@@ -365,16 +498,39 @@ export function localTransport({ seed, key = "clay-live:" + window.location.path
     let ops;
     try {
       ops = frame.ops.map(validateOp);
-      applyAll(m.value, ops).undo();
     } catch (err) {
       return { status: 422, body: { error: err.message }, retryAfter: 0 };
     }
+    const host = room.host;
+    const admission = frame.name === "page" && isYValue(m.value) ? admitPageOps({ y: m.value.y, ops, state: { ...host, compaction: frame.compaction }, limits: COMPACTION }) : null;
+    if (admission?.error) {
+      if (admission.error === "checkpoint-required") due();
+      return refused(admission.error, admission.retryAfter);
+    }
+    let trial;
+    try {
+      trial = applyAll(m.value, ops);
+    } catch (err) {
+      return { status: 422, body: { error: err.message }, retryAfter: 0 };
+    }
+    const bytes = admission ? pageBytes(trial.value) : 0;
+    trial.undo();
+    if (admission && bytes > admission.limit) {
+      if (admission.failed || (host.lastFull && host.deletesSince === 0)) return refused("page-full");
+      due(true);
+      return refused("checkpoint-required");
+    }
     room.seq += 1;
     room.lastN.set(nKey, { n: frame.n, seq: room.seq });
     const data = { epoch: room.epoch, seq: room.seq, name: frame.name, ops, from, n: frame.n };
     advance(data);
     post({ type: "data", data });
     handlers.frame(cloneJson(data));
+    if (admission) {
+      host.deletesSince += admission.deleted;
+      if (!admission.failed && bytes > COMPACTION.activeBytes) due(true);
+      else if (!admission.failed && admission.shape.loose + admission.added >= COMPACTION.soft) due();
+    }
     return { status: 200, body: { seq: room.seq, epoch: room.epoch }, retryAfter: 0 };
   }
 
@@ -416,6 +572,7 @@ export function localTransport({ seed, key = "clay-live:" + window.location.path
     }
     if (value === null || typeof value !== "object" || Array.isArray(value) !== Array.isArray(m.value)) return null;
     const file = { seq: seq ?? null, hash: textHash(serializeLive(value)) };
+    if (name === "page" && isYValue(m.value) && isYValue(value)) return mergePageFile(m, value, file);
     // A stamp of this room places the file. Still the value its stamp names, it
     // is a capture and holds no merge past the stamp; edited, or unstamped, it
     // descends from the file before it, which holds the merges in inFile.
@@ -456,6 +613,40 @@ export function localTransport({ seed, key = "clay-live:" + window.location.path
     return { changed, moved: true };
   }
 
+  // A save of the page merges as a union: every key whose content the room cannot
+  // rebuild from its own keys goes in under a fresh name of this room's, never the
+  // file's, so a key the room retired cannot come back through it. The merge is a
+  // move like any other, and the room is due a compaction after it.
+  function mergePageFile(m, value, file) {
+    if (value.g !== m.value.g) return null;
+    const host = room.host;
+    const adds = unionFileEntries({
+      y: m.value.y,
+      fileY: value.y,
+      retired: host.retired,
+      rangesFor: host.ranges,
+      keyFor: (i) => `f${textHash(room.epoch).slice(0, 8)}-${(room.seq + 1).toString(36)}-${i.toString(36)}`,
+    });
+    m.file = file;
+    const taken = new Set();
+    const ops = [];
+    for (const [key, v] of adds) {
+      if (Object.hasOwn(m.value.y, key) || taken.has(key)) continue;
+      taken.add(key);
+      ops.push(validateOp({ op: "set", path: ["y", key], value: v }));
+    }
+    if (!ops.length) return { changed: false, moved: false };
+    m.value = applyAll(m.value, ops).value;
+    room.seq += 1;
+    mirror.seq = room.seq;
+    m.last = room.seq;
+    pushLog(m, { seq: room.seq, ops });
+    m.inFile = [...m.inFile.filter((seq) => seq > m.floor), room.seq];
+    host.lastFull = null;
+    due();
+    return { changed: true, moved: true };
+  }
+
   // A file frame (another tab's or device's save, or this browser's own
   // checkpoint coming back), per saved store: { text, at, seq }. The sequencer
   // merges it; every other tab keeps the newest, for the room it may take.
@@ -529,7 +720,7 @@ export function localTransport({ seed, key = "clay-live:" + window.location.path
   }
 
   function onMessage({ data: m }) {
-    if (stopped || !m || m.v !== VERSION) return;
+    if (stopped || !m || m.version !== VERSION) return;
     if (room && ((m.type === "data" && m.data?.epoch === room.epoch) || ((m.type === "data-state" || m.type === "closed") && m.state?.epoch === room.epoch && m.state.writer !== clientId))) {
       forked();
       return;
@@ -590,7 +781,7 @@ export function localTransport({ seed, key = "clay-live:" + window.location.path
     if (!covered) return { status: "refused", code: "uncovered", ...held };
     closeRoom();
     unsubscribe?.();
-    laneKey = want ? `${key}:${FORMAT_SCOPE}` : key;
+    laneKey = scopeOf(want);
     scopeFloor = want;
     channel = localChannel(laneKey);
     unsubscribe = channel.subscribe("durable", (data) => onMessage({ data }));
@@ -632,7 +823,7 @@ export function localTransport({ seed, key = "clay-live:" + window.location.path
       };
     },
     async send(_clientId, frame) {
-      if (room) return roomApply(clientId, frame);
+        if (room) return roomApply(clientId, frame);
       return (await request({ type: "post", frame, epoch: mirror.epoch })) || { status: 0, body: null, retryAfter: 0 };
     },
     // The sequencer's state, installed into the mirror first. An answer to a

package/src/live/ops.js

@@ -318,6 +318,22 @@ export function serializeLive(value) {
   return JSON.stringify(value).replace(/</g, "\\u003c");
 }
 
+// True exactly when serializeLive gives both values the same text: the same keys in the
+// same order and equal leaves, read without writing either value out.
+export function sameLive(a, b) {
+  if (a === b) return true;
+  if (!isContainer(a) || !isContainer(b) || Array.isArray(a) !== Array.isArray(b)) return false;
+  if (Array.isArray(a)) {
+    if (a.length !== b.length) return false;
+    for (let i = 0; i < a.length; i++) if (!sameLive(a[i], b[i])) return false;
+    return true;
+  }
+  const keys = Object.keys(a), other = Object.keys(b);
+  if (keys.length !== other.length) return false;
+  for (let i = 0; i < keys.length; i++) if (keys[i] !== other[i] || !sameLive(a[keys[i]], b[keys[i]])) return false;
+  return true;
+}
+
 // A live tag's lineage, `live-at="<epoch>:<seq>:<hash>"`: the room position its
 // value was written at, and a hash of the exact text, so a room can tell its own
 // unedited value from an edit of it.

package/src/live/page-compaction.js

@@ -0,0 +1,162 @@
+import * as Y from '../vendor/yjs.vendor.js';
+import { toBase64, fromBase64 } from '../vendor/yjs.vendor.js';
+import { PART_KEY, isBase64, entriesOf } from '../page/ybind/parts.js';
+import { rangesOf, unite, within, docRanges } from '../page/ybind/containment.js';
+
+export const COMPACTION = Object.freeze({
+  version: 1,
+  loose: 32,
+  soft: 16,
+  activeBytes: 3840 * 1024,
+  deleteReserve: 64 * 1024,
+  newPerFrame: 4,
+  tailFloor: 64 * 1024,
+  leaseMs: 60 * 1000,
+  retired: 8192,
+});
+
+export const ROOM_ERROR_STATUS = Object.freeze({
+  'checkpoint-required': 503, 'page-full': 409, 'compaction-failed': 409,
+  'page-immutable': 409, 'page-batch': 409, 'page-update-invalid': 422, 'page-upgrade': 404,
+});
+
+export const CLIENT_KEY = /^[ur][0-9a-z]+(-[0-9a-z]+){2}$/;
+export const HOST_KEY = /^[ctf][0-9a-z]{8}-[0-9a-z]+(-[0-9a-z]+)?$/;
+const CHECKPOINT_KEY = /^(seed|c[0-9a-z]{8}-[0-9a-z]+)$/;
+
+export const isCheckpointKey = (key) => CHECKPOINT_KEY.test(key);
+
+export function pageShape(y, limits = COMPACTION) {
+  let checkpoints = 0, loose = 0, parts = 0, legacy = 0;
+  for (const key of Object.keys(y)) {
+    if (PART_KEY.test(key)) { parts++; continue; }
+    if (isCheckpointKey(key)) checkpoints++;
+    else loose++;
+    if (!CLIENT_KEY.test(key) && !HOST_KEY.test(key) && key !== 'seed') legacy++;
+  }
+  return { checkpoints, loose, parts, legacy, normal: parts === 0 && checkpoints <= 1 && loose <= limits.loose };
+}
+
+export function planCompaction(y, { overflow = false } = {}, limits = COMPACTION) {
+  const shape = pageShape(y, limits);
+  const entries = entriesOf(y);
+  const checkpoint = entries.filter(([key]) => isCheckpointKey(key));
+  const loose = entries.filter(([key]) => !isCheckpointKey(key));
+  const looseBytes = loose.reduce((sum, [, v]) => sum + v.length, 0);
+  const checkpointBytes = checkpoint.reduce((sum, [, v]) => sum + v.length, 0);
+  const full = !shape.normal || overflow || looseBytes > Math.max(limits.tailFloor, checkpointBytes / 4);
+  if (full) return { kind: 'full', sources: entries };
+  if (loose.length < 2 || shape.loose < limits.soft) return null;
+  return { kind: 'tail', sources: loose };
+}
+
+export function buildCandidate({ kind, sources, encode = Y.encodeStateAsUpdateV2 }) {
+  const updates = sources.map(([, v]) => fromBase64(v));
+  if (kind === 'tail') {
+    const merged = Y.mergeUpdatesV2(updates);
+    const index = rangesOf(merged);
+    for (const update of updates) if (!within(index, rangesOf(update))) return { failure: 'not-covered' };
+    return { update: toBase64(merged) };
+  }
+  const doc = new Y.Doc({ gc: true });
+  try {
+    for (const update of updates) Y.applyUpdateV2(doc, update);
+    if (doc.store.pendingStructs || doc.store.pendingDs) {
+      const merged = Y.mergeUpdatesV2(updates);
+      const index = rangesOf(merged);
+      for (const update of updates) if (!within(index, rangesOf(update))) return { failure: 'not-covered' };
+      return { update: toBase64(merged), pending: true };
+    }
+    const index = docRanges(doc);
+    for (const update of updates) if (!within(index, rangesOf(update))) return { failure: 'not-covered' };
+    const out = toBase64(encode(doc));
+    const check = new Y.Doc({ gc: true });
+    try {
+      Y.applyUpdateV2(check, fromBase64(out));
+      const stored = docRanges(check);
+      if (!stored.complete) return { failure: 'not-covered' };
+      for (const update of updates) if (!within(stored, rangesOf(update))) return { failure: 'not-covered' };
+    } finally { check.destroy(); }
+    return { update: out };
+  } catch (error) {
+    return { failure: 'invalid-update', message: error?.message || String(error) };
+  } finally {
+    doc.destroy();
+  }
+}
+
+export function checkUpdate(text) {
+  if (typeof text !== 'string' || !isBase64(text)) return null;
+  try {
+    const { items, deletes } = rangesOf(fromBase64(text));
+    let deleted = 0;
+    for (const spans of deletes.values()) for (const [from, to] of spans) deleted += to - from;
+    return { deleted, inserts: items.size > 0 };
+  } catch {
+    return null;
+  }
+}
+
+export function admitPageOps({ y, ops, state, limits }) {
+  if (state.compaction !== limits.version) return { error: 'page-upgrade' };
+  let added = 0;
+  let deleted = 0;
+  let inserts = false;
+  for (const op of ops) {
+    const key = op.path?.[1];
+    if (op.op !== 'set' || typeof key !== 'string' || !CLIENT_KEY.test(key)) return { error: 'page-immutable' };
+    if (Object.hasOwn(y, key)) {
+      if (y[key] !== op.value) return { error: 'page-immutable' };
+      continue;
+    }
+    if (state.retired.has(key)) return { error: 'page-immutable' };
+    const checked = checkUpdate(op.value);
+    if (!checked) return { error: 'page-update-invalid' };
+    added += 1;
+    deleted += checked.deleted;
+    inserts ||= checked.inserts;
+  }
+  if (added > limits.newPerFrame) return { error: 'page-batch' };
+  // A page whose compaction failed keeps taking edits up to its byte limit, as a room
+  // without compaction does, and refuses past it as full instead of waiting for a fold.
+  if (state.failed) return { added, deleted, inserts, limit: limits.activeBytes + (inserts ? 0 : limits.deleteReserve), shape: null, failed: true };
+  const shape = pageShape(y, limits);
+  if (!shape.normal || shape.loose + added > limits.loose) return { error: 'checkpoint-required', retryAfter: 0 };
+  return { added, deleted, inserts, limit: limits.activeBytes + (inserts ? 0 : limits.deleteReserve), shape };
+}
+
+export function contained(roomEntries, fileValue, rangesFor) {
+  const index = unite(roomEntries.map(([k, v]) => rangesFor(k, v)));
+  return within(index, rangesOf(fromBase64(fileValue)));
+}
+
+const UNIQUE_KEY = (key) => CLIENT_KEY.test(key) || HOST_KEY.test(key);
+
+export function createRangeCache(max = 256) {
+  const cache = new Map();
+  return (key, value) => {
+    const hit = cache.get(key);
+    if (hit && hit.value === value) return hit.ranges;
+    const ranges = rangesOf(fromBase64(value));
+    cache.delete(key);
+    cache.set(key, { value, ranges });
+    if (cache.size > max) cache.delete(cache.keys().next().value);
+    return ranges;
+  };
+}
+
+export function unionFileEntries({ y, fileY, retired, rangesFor, keyFor = (i, key) => key }) {
+  const roomEntries = entriesOf(y);
+  const have = new Map(roomEntries);
+  let index = null;
+  const adds = [];
+  for (const [key, value] of entriesOf(fileY ?? {})) {
+    if (have.get(key) === value) continue;
+    if (retired.has(key) && UNIQUE_KEY(key)) continue;
+    let ranges;
+    try { ranges = rangesOf(fromBase64(value)); } catch { continue; }
+    index ??= unite(roomEntries.map(([k, v]) => rangesFor(k, v)));
+    if (!within(index, ranges)) adds.push([key, value]);
+  }
+  return adds.map(([key, value], i) => [keyFor(i, key), value]);
+}

package/src/live/server-lane.js

@@ -46,6 +46,7 @@ export async function fetchLiveState(clientId, since = null, { signal } = {}) {
   const url = dataUrl();
   if (clientId) url.searchParams.set("client-id", clientId);
   if (since && typeof since.epoch === "string" && Number.isInteger(since.seq)) url.searchParams.set("since", `${since.epoch}:${since.seq}`);
+  url.searchParams.set("compaction", "1");
   try {
     const response = await fetch(url.href, {
       credentials: "same-origin",

package/src/live/store.js

@@ -4,7 +4,7 @@
 // Local ops change state at once. When authoritative moves, state is rebuilt in
 // place, so an object or list page code holds a reference to stays live.
 
-import { validateOp, checkOp, applyOp, applyOps, applyTracked, undoTracked, cloneJson, assignDeep, coalesce, overlaps, serializeLive } from "./ops.js";
+import { validateOp, checkOp, applyOp, applyOps, applyTracked, undoTracked, cloneJson, assignDeep, coalesce, overlaps, serializeLive, sameLive } from "./ops.js";
 
 const MUTATORS = new Set(["push", "pop", "shift", "unshift", "splice", "sort", "reverse", "fill", "copyWithin"]);
 const INDEX = /^(0|[1-9]\d*)$/;
@@ -112,6 +112,8 @@ export function createStore({ name, value, ephemeral = false }) {
   if (!isContainer(value)) throw new TypeError(`clay.live("${name}"): the tag must hold a JSON object or list`);
 
   let authoritative = cloneJson(value);
+  // The last room a rebase described, for a listener that subscribes after the lane joined.
+  let lastRoom = null;
   const state = cloneJson(value);
   // state is authoritative with pending replayed over it. Only a confirm whose
   // ops no longer apply breaks that, until the next rebuild.
@@ -171,7 +173,8 @@ export function createStore({ name, value, ephemeral = false }) {
       console.warn(`clay.live("${name}"): ignored an incoming value that is not ${kind(state)}`);
       return false;
     }
-    const before = serializeLive(state);
+    if (room) lastRoom = room;
+    const before = cloneJson(state);
     const prev = authoritative;
     authoritative = cloneJson(next);
     if (drop) pending = pending.filter((entry) => entry.id < drop[0] || entry.id > drop[1]);
@@ -179,7 +182,7 @@ export function createStore({ name, value, ephemeral = false }) {
     else pending = through === undefined ? pending.slice(confirmed) : pending.filter((entry) => entry.id > through);
     rebuild();
     tell({ source, frame, room, cause, prev, before: (path) => readPath(prev, path), next: authoritative });
-    if (serializeLive(state) !== before) emit(source, null);
+    if (!sameLive(before, state)) emit(source, null);
     return true;
   }
 
@@ -361,6 +364,9 @@ export function createStore({ name, value, ephemeral = false }) {
     get authoritative() {
       return authoritative;
     },
+    get room() {
+      return lastRoom;
+    },
     get pending() {
       return pending.map((entry) => entry.op);
     },
@@ -374,6 +380,7 @@ export function createStore({ name, value, ephemeral = false }) {
     },
     setMode(next) {
       mode = next;
+      if (next !== "fast") lastRoom = null;
     },
     // A transport hears every local op batch, in order, as it is committed.
     onLocal(fn) {
@@ -471,6 +478,7 @@ export function createStore({ name, value, ephemeral = false }) {
         }
         return readPath(authoritative, path);
       };
+      if (room) lastRoom = room;
       tell({ source, frame, room, cause, get prev() { return prev(); }, before, next: authoritative });
       if (changed) emit(source, null);
       return true;

package/src/loader-logic.js

@@ -214,7 +214,7 @@ const DRAFT_DROPS_CORE = ["attrs/refetch-on-save.js", "core/conflict-notice.js"]
 // Only that host's copies auto-add the update check; another source names it.
 const TEMPLATE_SOURCE_PREFIX = "https://hyperclay.com/templates/";
 
-export function resolveModules(params, isEditMode, { isDraftMode = false, hasSap = false, canEdit = false, templateSource = null } = {}) {
+export function resolveModules(params, isEditMode, { isDraftMode = false, hasSap = false, hasVariants = false, canEdit = false, templateSource = null } = {}) {
   const core = [...CORE_WAVES.always];
   if (isEditMode) core.push(...CORE_WAVES.editOnly);
 
@@ -228,6 +228,8 @@ export function resolveModules(params, isEditMode, { isDraftMode = false, hasSap
     if (spec.default) enabled.add(name);
   }
   if (hasSap) enabled.add("derive");
+  // Two or more looks need the dock's variant pill, and only someone who can edit sees it.
+  if (hasVariants && (isEditMode || canEdit)) enabled.add("menu");
   // ai-edit cannot run without the wire, so excluding the wire excludes it too.
   if (excluded.has("wire")) enabled.delete("ai-edit");
   if (!isEditMode && !isDraftMode && enabled.has("page")) enabled.add("page-view");