@panphora/clayjs 1.13.1 → 1.15.0

package/src/live/local-lane.js

@@ -18,7 +18,7 @@
 // where in the log it starts, the moves after that go back on top, and the
 // merge itself is logged as a diff so a later file stamped before it keeps it.
 
-import { applyOps, applyOp, checkOp, cloneJson, validateOp, serializeLive, textHash, diffOps } from "./ops.js";
+import { applyOp, applyTracked, checkOp, cloneJson, undoTracked, validateOp, serializeLive, textHash, diffOps } from "./ops.js";
 import { HyperMorph } from "../vendor/hyper-morph.vendor.js";
 import { movesSince } from "./moves.js";
 
@@ -28,6 +28,28 @@ export const CLOSE_WAIT_MS = 500;
 export const LOG_ENTRIES = 1024;
 export const LOG_BYTES = 256 * 1024;
 
+// ops applied to value in place, all or none: a throw takes back the ones
+// before it. Returns the value to go on with, and how to take the ops back.
+function applyAll(value, ops) {
+  const done = [];
+  const undo = () => {
+    for (let i = done.length - 1; i >= 0; i--) undoTracked(done[i]);
+  };
+  try {
+    for (const op of ops) {
+      const rec = applyTracked(value, op);
+      if (rec) {
+        done.push(rec);
+        value = rec.root;
+      }
+    }
+  } catch (err) {
+    undo();
+    throw err;
+  }
+  return { value, undo };
+}
+
 const encoder = new TextEncoder();
 
 export function localLaneSupported() {
@@ -99,7 +121,7 @@ export function localTransport({ seed, key = "clay-live:" + window.location.path
   function advance(data) {
     const m = mirror.stores.get(data.name);
     if (m) {
-      m.value = applyOps(cloneJson(m.value), data.ops);
+      m.value = applyAll(m.value, data.ops).value;
       m.last = data.seq;
       if (!m.ephemeral) pushLog(m, { seq: data.seq, ops: data.ops });
     }
@@ -119,7 +141,10 @@ export function localTransport({ seed, key = "clay-live:" + window.location.path
 
   const landedIn = (m) => (m.behind || m.landed === null ? -1 : m.landed);
 
-  function roomState(since = null) {
+  // A client's own read (since is its { epoch, seq }, or null for a first read)
+  // carries moves, as the host's state() does: those after its seq in its own
+  // epoch, else this epoch's from its start. A broadcast passes no since.
+  function roomState(since) {
     const stores = {};
     const meta = {};
     for (const [name, m] of mirror.stores) {
@@ -127,9 +152,10 @@ export function localTransport({ seed, key = "clay-live:" + window.location.path
       meta[name] = { log: m.log, last: m.last, landed: m.landed, floor: m.floor, behind: m.behind, file: m.file, inFile: m.inFile };
     }
     const state = { epoch: room.epoch, seq: room.seq, writer: clientId, stores, mirror: { seen: [...mirror.seen], stores: meta } };
-    if (since && since.epoch === mirror.epoch) {
+    if (since !== undefined) {
+      const from = since && since.epoch === mirror.epoch && Number.isInteger(since.seq) ? since.seq : 0;
       const moves = {};
-      for (const [name, m] of mirror.stores) moves[name] = movesSince(m, since);
+      for (const [name, m] of mirror.stores) moves[name] = movesSince(m, { seq: from });
       state.moves = moves;
     }
     return state;
@@ -322,7 +348,7 @@ export function localTransport({ seed, key = "clay-live:" + window.location.path
     let ops;
     try {
       ops = frame.ops.map(validateOp);
-      applyOps(cloneJson(m.value), ops);
+      applyAll(m.value, ops).undo();
     } catch (err) {
       return { status: 422, body: { error: err.message }, retryAfter: 0 };
     }

package/src/live/ops.js

@@ -176,6 +176,83 @@ export function applyOps(root, ops) {
   return value;
 }
 
+// applyOp, keeping what it changed so a caller can size the change or take it
+// back. null when the op changes nothing. Otherwise root is the value to go on
+// with (a new one only for a whole-value set, where at is null and old is the
+// previous root), and at[key] is the one slot the op changed, at path. kind
+// says how: "put" replaced old, "add" made the slot (a new key, or an item
+// appended to a list of size items), "remove" took old out. For an object, keys
+// lists its keys before a remove, and empty says it had none before an add.
+// Checks the op first, so a throw changes nothing.
+export function applyTracked(root, op) {
+  checkOp(root, op);
+  const { path } = op;
+  if (path.length === 0) {
+    return { root: cloneJson(op.value), at: null, key: null, kind: "put", old: root, path: [], size: 0, keys: null, empty: false };
+  }
+  let at = root;
+  let depth = path.length - 1;
+  for (let i = 0; i < path.length - 1; i++) {
+    const next = read(at, path[i]);
+    if (!isContainer(next)) {
+      if (op.op === "del") return null;
+      depth = i;
+      break;
+    }
+    at = next;
+  }
+  const list = Array.isArray(at);
+  const key = list ? indexOf(at, path[depth]) : String(path[depth]);
+  const size = list ? at.length : 0;
+  const had = list ? key < size : has(at, key);
+  if (op.op === "del" && !had) return null;
+  const kind = op.op === "del" ? "remove" : had ? "put" : "add";
+  const rec = { root, at, key, kind, old: had ? at[key] : undefined, path: path.slice(0, depth + 1), size, keys: null, empty: false };
+  if (!list && kind === "remove") rec.keys = Object.keys(at);
+  if (!list && kind === "add") rec.empty = isEmpty(at);
+  applyOp(root, op);
+  return rec;
+}
+
+function isEmpty(obj) {
+  for (const key in obj) if (has(obj, key)) return false;
+  return true;
+}
+
+// Takes back one applyTracked change; a frame's records go back newest first.
+// Returns the root as it was before that op.
+export function undoTracked(rec) {
+  const { at, key, kind, old } = rec;
+  if (at === null) return old;
+  if (Array.isArray(at)) {
+    if (kind === "put") at[key] = old;
+    else if (kind === "add") at.length = rec.size;
+    else at.splice(key, 0, old);
+  } else if (kind === "put") setOwn(at, key, old);
+  else if (kind === "add") delete at[key];
+  else if (rec.keys[rec.keys.length - 1] === key) setOwn(at, key, old);
+  else {
+    // Back in its place: every key after it is put back behind it.
+    const values = rec.keys.map((k) => (k === key ? old : at[k]));
+    for (const k of rec.keys) if (k !== key) delete at[k];
+    rec.keys.forEach((k, i) => setOwn(at, k, values[i]));
+  }
+  return rec.root;
+}
+
+// What one applyTracked change did to the length of serializeLive(root), with
+// size(v) the length of serializeLive(v) in the caller's unit and total the
+// root's length before. Call it before the next op touches the slot.
+export function sizeChange(rec, size, total) {
+  const { at, key, kind, old } = rec;
+  if (at === null) return size(rec.root) - total;
+  const list = Array.isArray(at);
+  const slot = list ? 0 : size(key) + 1;
+  if (kind === "put") return size(at[key]) - size(old);
+  if (kind === "add") return slot + size(at[key]) + ((list ? rec.size > 0 : !rec.empty) ? 1 : 0);
+  return -(slot + size(old)) - ((list ? rec.size : rec.keys.length) > 1 ? 1 : 0);
+}
+
 const samePath = (a, b) => a.length === b.length && a.every((s, i) => String(s) === String(b[i]));
 
 // Merges neighbouring ops on the same path, without changing the result:

package/src/live/provenance.js

@@ -0,0 +1,129 @@
+const CLIENT = /^[A-Za-z0-9_-]{1,64}$/;
+const NUMBER = /^[1-9]\d{0,15}$/;
+export const RANGE_LIMIT = 256;
+
+const integer = (text) => NUMBER.test(text) && Number.isSafeInteger(Number(text));
+
+export function unionRanges(...sets) {
+  const clients = new Map();
+  for (const set of sets) for (const [client, ranges] of set ?? []) {
+    if (!clients.has(client)) clients.set(client, []);
+    clients.get(client).push(...ranges.map(([lo, hi]) => [lo, hi]));
+  }
+  return [...clients].sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([client, ranges]) => {
+    const out = [];
+    for (const [lo, hi] of ranges.sort((a, b) => a[0] - b[0])) {
+      const last = out[out.length - 1];
+      if (last && lo <= last[1] + 1) last[1] = Math.max(last[1], hi);
+      else out.push([lo, hi]);
+    }
+    return [client, out];
+  }).filter(([, ranges]) => ranges.length);
+}
+
+export const rangeCount = (set) => set.reduce((n, [, ranges]) => n + ranges.length, 0);
+export const hasNumber = (ranges, n) => !!ranges?.some(([lo, hi]) => lo <= n && n <= hi);
+export const contains = (set, client, n) => hasNumber(set?.find(([id]) => id === client)?.[1], n);
+export const identitySet = ([client, n]) => [[client, [[n, n]]]];
+export const frameSet = (by) => by?.v === 2 ? by.frames : by ? [[by.client, [[1, by.n]]]] : [];
+export const descendants = (by) => by?.v === 2 ? unionRanges(by.ancestors, identitySet(by.capture)) : [];
+
+function parseRanges(text) {
+  if (!text) return [];
+  const out = [];
+  let count = 0;
+  for (const part of text.split(';')) {
+    const fields = part.split(':');
+    if (fields.length !== 2 || !CLIENT.test(fields[0]) || !fields[1]) return null;
+    const ranges = [];
+    for (const raw of fields[1].split(',')) {
+      if (++count > RANGE_LIMIT) return null;
+      const ends = raw.split('-');
+      if (ends.length > 2 || !ends.every(integer)) return null;
+      const lo = Number(ends[0]), hi = Number(ends[1] ?? ends[0]);
+      if (hi < lo) return null;
+      ranges.push([lo, hi]);
+    }
+    out.push([fields[0], ranges]);
+  }
+  return unionRanges(out);
+}
+
+function printRanges(set) {
+  return unionRanges(set).map(([client, ranges]) => `${client}:${ranges.map(([lo, hi]) => lo === hi ? String(lo) : `${lo}-${hi}`).join(',')}`).join(';');
+}
+
+export function parseBy(text) {
+  if (typeof text !== 'string' || text.length > 32768) return null;
+  if (!text.startsWith('v2|')) {
+    const m = /^([A-Za-z0-9_-]{1,64}):([1-9]\d{0,15})$/.exec(text);
+    return m && integer(m[2]) ? { client: m[1], n: Number(m[2]) } : null;
+  }
+  const parts = text.split('|');
+  if (parts.length !== 4) return null;
+  const frames = parseRanges(parts[1]), ancestors = parseRanges(parts[2]);
+  const capture = /^([A-Za-z0-9_-]{1,64}):([1-9]\d{0,15})$/.exec(parts[3]);
+  if (!frames || !ancestors || !capture || !integer(capture[2])) return null;
+  if (rangeCount(frames) + rangeCount(ancestors) + 1 > RANGE_LIMIT) return null;
+  const id = [capture[1], Number(capture[2])];
+  if (contains(ancestors, ...id)) return null;
+  return { v: 2, frames, ancestors, capture: id };
+}
+
+export function formatBy(by) {
+  const text = `v2|${printRanges(by.frames)}|${printRanges(by.ancestors)}|${by.capture[0]}:${by.capture[1]}`;
+  return parseBy(text) ? text : null;
+}
+
+export function receiptHas(receipt, n) {
+  return !!receipt && (n <= receipt.n || hasNumber(receipt.saved, n));
+}
+
+export function receiptBytes(receipt) {
+  return receipt?.saved ? JSON.stringify({ saved: receipt.saved, savedSeq: receipt.savedSeq }).length : 0;
+}
+
+export function replaceReceipt(room, key, next) {
+  const delta = receiptBytes(next) - receiptBytes(room.lastN.get(key));
+  room.receiptBytes = (room.receiptBytes ?? 0) + delta;
+  room.lastN.delete(key);
+  room.lastN.set(key, next);
+  return delta;
+}
+
+export function retireReceipt(room, key) {
+  const bytes = receiptBytes(room.lastN.get(key));
+  room.lastN.delete(key);
+  room.receiptBytes = (room.receiptBytes ?? 0) - bytes;
+  room.receiptGeneration = (room.receiptGeneration ?? 0) + 1;
+  return -bytes;
+}
+
+export function saveReceipt(room, key, ranges, seq, limit) {
+  const last = room.lastN.get(key);
+  if (!last && room.lastN.size >= limit) {
+    retireReceipt(room, room.lastN.keys().next().value);
+  }
+  const next = last ? { ...last } : { n: 0, seq: 0 };
+  const kept = unionRanges([['r', next.saved ?? []]], [['r', ranges]])[0]?.[1]
+    .filter(([, hi]) => hi > next.n).map(([lo, hi]) => [Math.max(lo, next.n + 1), hi]) ?? [];
+  if (kept.length > RANGE_LIMIT) {
+    retireReceipt(room, key);
+    next.saved = kept.slice(-RANGE_LIMIT);
+    next.savedSeq = Math.max(next.savedSeq ?? 0, seq);
+  } else {
+    if (kept.length) {
+      next.saved = kept;
+      next.savedSeq = Math.max(next.savedSeq ?? 0, seq);
+    } else {
+      delete next.saved;
+      delete next.savedSeq;
+    }
+  }
+  replaceReceipt(room, key, next);
+}
+
+export function postReceipt(last, n, seq) {
+  const saved = (last?.saved ?? []).filter(([, hi]) => hi > n).map(([lo, hi]) => [Math.max(lo, n + 1), hi]);
+  return { n, seq, ...(saved.length && { saved, savedSeq: last.savedSeq }) };
+}

package/src/live/save-history.js

@@ -0,0 +1,188 @@
+export const RECOVERY_WINDOW_MS = 45_000;
+export const RECOVERY_ROUNDS_MS = [0, 10_000, 30_000];
+export const QUERY_TIMEOUT_MS = 6_000;
+export const MAX_HISTORY_IDS = 2;
+
+export async function sha256Hex(html) {
+  const subtle = globalThis.crypto && globalThis.crypto.subtle;
+  if (!subtle) return null;
+  try {
+    const bytes = new TextEncoder().encode(String(html));
+    const digest = await subtle.digest("SHA-256", bytes);
+    const view = new Uint8Array(digest);
+    let hex = "";
+    for (let i = 0; i < view.length; i += 1) hex += view[i].toString(16).padStart(2, "0");
+    return hex;
+  } catch (_) {
+    return null;
+  }
+}
+
+function uniqueSaveIds(attempts) {
+  const ids = [];
+  for (const attempt of attempts || []) {
+    const saveId = attempt && attempt.saveId;
+    if (typeof saveId === "string" && saveId && !ids.includes(saveId)) ids.push(saveId);
+    if (ids.length >= MAX_HISTORY_IDS) break;
+  }
+  return ids;
+}
+
+function positive(answer, saveId, requestHash) {
+  return Boolean(
+    answer &&
+      typeof answer === "object" &&
+      answer.saveId === saveId &&
+      answer.requestHash === requestHash &&
+      answer.status === "applied"
+  );
+}
+
+export function createSaveHistoryRecovery({
+  query,
+  digest = sha256Hex,
+  now = () => Date.now(),
+  setTimer = setTimeout,
+  clearTimer = clearTimeout,
+} = {}) {
+  function start({ captureId, html, attempts, initialOutcome, startedAt, onApplied, onUnknown }) {
+    const deadline = startedAt + RECOVERY_WINDOW_MS;
+    const ids = uniqueSaveIds(attempts);
+    const timers = new Set();
+    const controllers = new Set();
+    let body = html;
+    let hash = null;
+    let terminal = false;
+
+    function call(fn, arg) {
+      try {
+        if (typeof fn === "function") fn(arg);
+      } catch (err) {
+        console.error("clayjs: save-history callback failed:", err);
+      }
+    }
+
+    function finish() {
+      terminal = true;
+      body = null;
+      for (const timer of timers) clearTimer(timer);
+      timers.clear();
+      for (const controller of controllers) controller.abort();
+      controllers.clear();
+    }
+
+    function apply(saveId, requestHash) {
+      if (terminal) return;
+      finish();
+      call(onApplied, { captureId, saveId, requestHash });
+    }
+
+    function giveUp(reason) {
+      if (terminal) return;
+      finish();
+      call(onUnknown, { captureId, reason });
+    }
+
+    function addTimer(fn, delay) {
+      let id = null;
+      id = setTimer(() => {
+        if (id !== null) timers.delete(id);
+        fn();
+      }, Math.max(0, delay));
+      timers.add(id);
+      return id;
+    }
+
+    function dropTimer(id) {
+      if (id !== null) {
+        clearTimer(id);
+        timers.delete(id);
+      }
+    }
+
+    async function probe(saveId) {
+      if (terminal) return;
+      const remaining = deadline - now();
+      if (remaining <= 0) return;
+      const controller = new AbortController();
+      controllers.add(controller);
+      const timer = addTimer(() => controller.abort(), Math.min(QUERY_TIMEOUT_MS, remaining));
+      let answer = null;
+      try {
+        answer = await query({ saveId, requestHash: hash, signal: controller.signal });
+      } catch (_) {
+        answer = null;
+      } finally {
+        dropTimer(timer);
+        controllers.delete(controller);
+      }
+      if (terminal) return;
+      if (positive(answer, saveId, hash)) apply(saveId, hash);
+    }
+
+    function runRound() {
+      if (terminal) return;
+      if (now() >= deadline) return;
+      for (const saveId of ids) probe(saveId);
+    }
+
+    async function run() {
+      let computed = null;
+      try {
+        computed = await digest(body);
+      } catch (_) {
+        computed = null;
+      }
+      body = null;
+      if (terminal) return;
+      if (typeof computed !== "string" || !computed) {
+        giveUp("unavailable");
+        return;
+      }
+      hash = computed;
+
+      if (
+        initialOutcome &&
+        typeof initialOutcome === "object" &&
+        initialOutcome.status === "applied" &&
+        initialOutcome.requestHash === hash &&
+        ids.includes(initialOutcome.saveId)
+      ) {
+        apply(initialOutcome.saveId, hash);
+        return;
+      }
+
+      const elapsed = now() - startedAt;
+      if (elapsed >= RECOVERY_WINDOW_MS) {
+        giveUp("expired");
+        return;
+      }
+
+      let due = -1;
+      for (let i = 0; i < RECOVERY_ROUNDS_MS.length; i += 1) {
+        if (RECOVERY_ROUNDS_MS[i] <= elapsed) due = i;
+      }
+      if (due < 0) {
+        for (let i = 0; i < RECOVERY_ROUNDS_MS.length; i += 1) {
+          addTimer(runRound, startedAt + RECOVERY_ROUNDS_MS[i] - now());
+        }
+        return;
+      }
+      runRound();
+      for (let i = due + 1; i < RECOVERY_ROUNDS_MS.length; i += 1) {
+        addTimer(runRound, startedAt + RECOVERY_ROUNDS_MS[i] - now());
+      }
+    }
+
+    addTimer(() => giveUp("expired"), deadline - now());
+    run();
+
+    return {
+      cancel() {
+        finish();
+      },
+    };
+  }
+
+  return { start };
+}

package/src/live/server-lane.js

@@ -39,7 +39,7 @@ export async function relayData(clientId, frame) {
  * The room's { epoch, seq, writer, stores }; { status } when the host refused;
  * null when the request never got an answer. clientId marks this tab as taking part.
  */
-export async function fetchLiveState(clientId, since = null) {
+export async function fetchLiveState(clientId, since = null, { signal } = {}) {
   const url = dataUrl();
   if (clientId) url.searchParams.set("client-id", clientId);
   if (since && typeof since.epoch === "string" && Number.isInteger(since.seq)) url.searchParams.set("since", `${since.epoch}:${since.seq}`);
@@ -48,8 +48,15 @@ export async function fetchLiveState(clientId, since = null) {
       credentials: "same-origin",
       cache: "no-store",
       headers: { "Document-URL": window.location.href },
+      signal,
     });
-    if (!response.ok) return { status: response.status };
+    if (!response.ok) {
+      let body = null;
+      try {
+        body = await response.json();
+      } catch {}
+      return { status: response.status, code: body?.code ?? null, retryAfter: Number(response.headers?.get?.("Retry-After")) || 0 };
+    }
     const state = await response.json();
     // The stream is open but the host has not seen its subscription yet: not
     // ready, and the lane reads again.