@panphora/clayjs 1.17.0 → 1.18.0

package/src/core/serialize-steps.js

@@ -0,0 +1,109 @@
+/**
+ * serialize-steps.js — the save's serializations as generators, for a document big enough
+ * to prepare its save across tasks.
+ *
+ * Each one returns exactly the string its one-call twin in snapshot.js returns, built child
+ * by child so it can yield between pieces. They live apart from snapshot.js because every
+ * page loads that at boot, and only the sliced save (core/save-job.js, a lazy chunk) and a
+ * sliced save's live-sync relay ever run these.
+ */
+
+import {
+  clonePreventingOnclone,
+  noscriptBlocks,
+  prepareForCompare,
+  swapNoscriptMarkers
+} from './snapshot.js';
+import { TAB_LOCAL_ROOT_ATTRS } from '../lib/root-attrs.js';
+
+const XHTML_NS = 'http://www.w3.org/1999/xhtml';
+
+// How much serialized output a sliced save writes before it hands the page back, and how
+// big a subtree has to be before it is serialized child by child instead of in one call.
+const SLICE_BYTES = 64 * 1024;
+const SLICE_MIN_ELEMENTS = 256;
+
+function sliceable(el) {
+  return el.namespaceURI === XHTML_NS
+    && el.localName !== 'template'
+    && el.getElementsByTagName('*').length >= SLICE_MIN_ELEMENTS;
+}
+
+/**
+ * The children of `el`, each one step, `yield`ing every `SLICE_BYTES` of output. A text
+ * or comment child is serialized inside `bare`, a childless copy of its parent, so its
+ * escaping follows the same parent element the one-call serializer would use.
+ */
+function* serializeChildrenSteps(el, bare, out, budget) {
+  for (const child of el.childNodes) {
+    if (child.nodeType === 1 && sliceable(child)) {
+      yield* serializeElementSteps(child, out, budget);
+      continue;
+    }
+    let piece;
+    if (child.nodeType === 1) piece = child.outerHTML;
+    else {
+      bare.replaceChildren(child.cloneNode(false));
+      piece = bare.innerHTML;
+    }
+    out.push(piece);
+    budget.bytes += piece.length;
+    if (budget.bytes >= SLICE_BYTES) {
+      budget.bytes = 0;
+      yield;
+    }
+  }
+}
+
+/**
+ * `el.outerHTML`, built from its open tag, each child's own serialization and its end
+ * tag, `yield`ing every `SLICE_BYTES` of output.
+ */
+function* serializeElementSteps(el, out, budget) {
+  const bare = clonePreventingOnclone(el, false);
+  const shell = bare.outerHTML;
+  const endTag = `</${bare.localName}>`;
+  out.push(shell.slice(0, shell.length - endTag.length));
+  yield* serializeChildrenSteps(el, bare, out, budget);
+  out.push(endTag);
+}
+
+/** `"<!DOCTYPE html>" + root.outerHTML`, in steps. */
+export function* serializeTreeSteps(root) {
+  const out = ["<!DOCTYPE html>"];
+  yield* serializeElementSteps(root, out, { bytes: 0 });
+  return out.join('');
+}
+
+/** `serializeClone`, in steps: the same bytes, `yield`ing every `SLICE_BYTES` of output. */
+export function* serializeCloneSteps(clone) {
+  const blocks = noscriptBlocks(clone);
+  if (!blocks.length) return yield* serializeTreeSteps(clone);
+  const raw = swapNoscriptMarkers(blocks);
+  let html;
+  try {
+    html = yield* serializeTreeSteps(clone);
+  } finally {
+    for (const [el, kids] of blocks) el.replaceChildren(...kids);
+  }
+  for (const [marker, text] of raw) html = html.split(marker).join(text);
+  return html;
+}
+
+/** `serializeForCompare`, in steps. */
+export function* serializeForCompareSteps(clone) {
+  prepareForCompare(clone);
+  return yield* serializeTreeSteps(clone);
+}
+
+/** `serializeForSync`, in steps: the same bytes, `yield`ing every `SLICE_BYTES` of output. */
+export function* serializeForSyncSteps(clone) {
+  const bareRoot = clonePreventingOnclone(clone, false);
+  for (const name of TAB_LOCAL_ROOT_ATTRS) bareRoot.removeAttribute(name);
+  const shell = bareRoot.outerHTML;
+  const endTag = `</${bareRoot.localName}>`;
+  const out = ["<!DOCTYPE html>", shell.slice(0, shell.length - endTag.length)];
+  yield* serializeChildrenSteps(clone, clonePreventingOnclone(clone, false), out, { bytes: 0 });
+  out.push(endTag);
+  return out.join('');
+}

package/src/core/snapshot.js

@@ -58,6 +58,7 @@ const snapshotProvenance = new WeakMap();
 // because many things want a turn; serializing the document is one decision, so a
 // second renderer would be two answers to "what bytes does this save send".
 let saveRenderer = null;
+let saveRendererSteps = null;
 
 // The capture identity for save captures. A refused save that may retry reads it
 // before sending and again before each retry: a capture newer than the one those
@@ -237,10 +238,16 @@ export function onSnapshot(callback) {
  * strings are only ever compared against each other, so they gain nothing from
  * source fidelity and would pay a full render on every keystroke's dirty check.
  *
+ * `steps` is the same render as a generator, for a save that verifies across tasks; a
+ * renderer without one is called synchronously at that point.
+ *
  * @param {?Function} renderer - (clone, today) => string, or null to restore outerHTML
+ * @param {Object} [options]
+ * @param {?Function} [options.steps] - (clone, today) => generator
  */
-export function setSaveRenderer(renderer) {
+export function setSaveRenderer(renderer, { steps = null } = {}) {
   saveRenderer = renderer;
+  saveRendererSteps = steps;
 }
 
 const XHTML_NS = 'http://www.w3.org/1999/xhtml';
@@ -255,7 +262,22 @@ const XHTML_NS = 'http://www.w3.org/1999/xhtml';
  * reads as literal text, and every later save escaped it again. The text is swapped for
  * a marker only while serializing, so the clone holds the same nodes when this returns.
  */
-function serializeClone(clone) {
+export function serializeClone(clone) {
+  const blocks = noscriptBlocks(clone);
+  if (!blocks.length) return "<!DOCTYPE html>" + clone.outerHTML;
+  const raw = swapNoscriptMarkers(blocks);
+  let html;
+  try {
+    html = "<!DOCTYPE html>" + clone.outerHTML;
+  } finally {
+    for (const [el, kids] of blocks) el.replaceChildren(...kids);
+  }
+  for (const [marker, text] of raw) html = html.split(marker).join(text);
+  return html;
+}
+
+/** Every <noscript> in the tree whose content is text only, with the children it held. */
+export function noscriptBlocks(clone) {
   const blocks = [];
   const collect = (node) => {
     for (const child of node.childNodes) {
@@ -269,21 +291,34 @@ function serializeClone(clone) {
     }
   };
   collect(clone);
-  if (!blocks.length) return "<!DOCTYPE html>" + clone.outerHTML;
+  return blocks;
+}
+
+/** Swap each block's text for a marker, and answer the text each marker stands for. */
+export function swapNoscriptMarkers(blocks) {
   const nonce = Math.random().toString(36).slice(2);
-  const raw = blocks.map(([el, kids], i) => {
+  return blocks.map(([el, kids], i) => {
     const marker = `\uE000clay-noscript-${nonce}-${i}\uE000`;
     el.replaceChildren(el.ownerDocument.createTextNode(marker));
     return [marker, kids.map((n) => n.data).join('')];
   });
-  let html;
+}
+
+/**
+ * Run one resume of a save renderer with `context` as the capture being serialized.
+ *
+ * The capture is never the active one while the task that is not running it looks: a
+ * renderer that verifies across tasks is resumed once per task, and each resume is its
+ * own call in here.
+ */
+export function runSaveRenderer(context, resume) {
+  const previous = activeCapture;
+  activeCapture = context;
   try {
-    html = "<!DOCTYPE html>" + clone.outerHTML;
+    return resume();
   } finally {
-    for (const [el, kids] of blocks) el.replaceChildren(...kids);
+    activeCapture = previous;
   }
-  for (const [marker, text] of raw) html = html.split(marker).join(text);
-  return html;
 }
 
 /**
@@ -303,15 +338,11 @@ function serializeSaveClone(clone, context = null) {
   }
   const today = serializeClone(clone);
   if (!saveRenderer) return today;
-  const previous = activeCapture;
-  activeCapture = context;
   try {
-    return saveRenderer(clone, today);
+    return runSaveRenderer(context, () => saveRenderer(clone, today));
   } catch (err) {
     console.error('clayjs: save renderer failed, sending the full serialization', err);
     return today;
-  } finally {
-    activeCapture = previous;
   }
 }
 
@@ -343,13 +374,26 @@ export function addDocumentTransform(callback) {
  *
  * @returns {HTMLElement} Cloned document element with snapshot hooks applied
  */
-function clonePreventingOnclone(node, deep = true) {
+export function clonePreventingOnclone(node, deep = true) {
   const prev = window.__preventOnclone;
   window.__preventOnclone = true;
   try { return node.cloneNode(deep); }
   finally { window.__preventOnclone = prev; }
 }
 
+// A clone OF the save clone (the comparison and dirty trees) is not the page, so the
+// view recorded no provenance for it: a transform that asks originalSnapshotNode would
+// get null there and answer differently than on the save clone it came from. The two
+// trees are isomorphic when this runs, so a lockstep walk registers each clone node
+// against the original its save-clone twin maps to.
+function carryProvenance(saveNode, cloneNode) {
+  const live = snapshotProvenance.get(saveNode);
+  if (live) snapshotProvenance.set(cloneNode, live);
+  const saveKids = saveNode.children;
+  const cloneKids = cloneNode.children;
+  for (let i = 0; i < saveKids.length; i++) carryProvenance(saveKids[i], cloneKids[i]);
+}
+
 export function captureSnapshot({ flushUndo = true, authored = true, forSave = false, requestRefresh = false, context = null } = {}) {
   // Force-close any pending undo idle batch BEFORE cloning the DOM, so the
   // snapshot reflects a clean undo boundary. Without this, a save that fires
@@ -473,12 +517,16 @@ export function captureSaveClone({ flushUndo = true, authored = true } = {}) {
 // check must not read that as an edit the person would lose on close.
 const EMPTY_INERT_ATTRS = ['style', 'class'];
 
-function serializeForCompare(clone) {
+export function prepareForCompare(clone) {
   stripDerived(clone);
   for (const name of EMPTY_INERT_ATTRS) {
     if (clone.getAttribute(name) === '') clone.removeAttribute(name);
     for (const el of clone.querySelectorAll(`[${name}=""]`)) el.removeAttribute(name);
   }
+}
+
+export function serializeForCompare(clone) {
+  prepareForCompare(clone);
   return "<!DOCTYPE html>" + clone.outerHTML;
 }
 
@@ -565,6 +613,7 @@ export function captureForComparisonAndDirty({ flushUndo = true } = {}) {
   const dirtyClone = clone.querySelector(NO_TRIGGER_AUTOSAVE_SELECTOR)
     ? clonePreventingOnclone(clone)
     : null;
+  if (dirtyClone) carryProvenance(clone, dirtyClone);
 
   for (const el of clone.querySelectorAll(STRIP_FROM_COMPARISON)) {
     el.remove();
@@ -593,23 +642,32 @@ export function captureForComparisonAndDirty({ flushUndo = true } = {}) {
 }
 
 /**
- * Single-capture function for both saving and comparison.
+ * Take one save capture's looking at the live page: in one synchronous call, everything
+ * that must see it (the snapshot, its hooks, snapshot-ready, the authored handlers, the
+ * comparison clones and every transform over all three).
  *
- * Clones the DOM once, then clones that clone for comparison.
- * More efficient than calling captureForSave() and captureForComparison() separately.
+ * What comes back is the pin the rest of the capture reads: every value that work needs,
+ * so a page edited afterwards cannot change the answer. It is the one place the strip and
+ * transform rules live, for both the one-task capture and a job that finishes across
+ * tasks — core/save-job.js, which only a save that prepares across tasks loads.
  *
  * @param {Object} options
  * @param {boolean} options.emitForSync - Whether to emit snapshot-ready event (default: true)
  * @param {boolean} options.forRequest - These bytes go to a host, so the renderer may
  *   attach the verified model they came from, for the accepted save to adopt
- * @returns {{ forSave: string, forComparison: string, forDirty: string, captureId: number }}
+ * @param {?{name: string, content: string}} options.pageFormatMarker - The document format
+ *   these bytes must declare, as the attributes of the marker element. It goes into the
+ *   save clone after the comparison domains are read and before the renderer verifies, so
+ *   the bytes verified are the bytes sent; the comparison baselines stay marker-free,
+ *   exactly as they do on the one-task path
+ * @returns {Object} the pin `job.steps()` and the one-task capture both read
  */
-export function captureForSaveAndComparison({ emitForSync = true, flushUndo = true, requestRefresh = false, forRequest = false } = {}) {
+export function prepareSaveCapture({ emitForSync = true, flushUndo = true, requestRefresh = false, forRequest = false, pageFormatMarker = null, sliced = false } = {}) {
   if (!Number.isSafeInteger(captureCounter + 1)) throw new Error('Save capture identity exhausted');
   const captureId = ++captureCounter;
   // The context this capture's renderer may attach a private artifact to, and the
   // record of whether the page has its own save handlers. It never reaches the object
-  // this function returns: that keeps its four public fields.
+  // the capture returns: that keeps its four public fields.
   //
   // Only for a capture whose bytes go to a host. A capture taken for anything else (a
   // live-sync disk base, a download, an inspection) has no request to adopt it, so it
@@ -619,14 +677,12 @@ export function captureForSaveAndComparison({ emitForSync = true, flushUndo = tr
 
   // Emit for live-sync before any stripping.
   //
-  // A listener gets the very clone this function goes on to serialize into
-  // forSave, forComparison and forDirty — that sharing is the point, it saves a
-  // second full DOM clone per save. So a listener may READ it and must not write
-  // to it: anything it changes lands in the saved bytes and in both baselines,
-  // and a baseline the dirty check cannot reproduce warns on close forever.
+  // A listener gets the very clone this capture goes on to serialize into forSave,
+  // forComparison and forDirty — that sharing is the point, it saves a second full DOM
+  // clone per save. So a listener may READ it and must not write to it.
   if (emitForSync) {
     document.dispatchEvent(new CustomEvent('clay:snapshot-ready', {
-      detail: { documentElement: clone, forSave: true, captureId }
+      detail: { documentElement: clone, forSave: true, captureId, sliced }
     }));
   }
 
@@ -635,15 +691,14 @@ export function captureForSaveAndComparison({ emitForSync = true, flushUndo = tr
 
   // Clone for comparison before stripping (cheaper than cloning live DOM)
   const compareClone = clonePreventingOnclone(clone);
+  carryProvenance(clone, compareClone);
 
-  // The dirty domain differs from the autosave domain only inside
-  // no-trigger-autosave regions, so a document without one pays nothing: the two
-  // strip sets remove exactly the same nodes and one string serves both. The
-  // predicate reads the captured clone after authored handlers have run, and
-  // covers all three spellings via the policy's own selector.
+  // The dirty domain differs from the autosave domain only inside no-trigger-autosave
+  // regions, so a document without one pays nothing: one string serves both.
   const dirtyClone = clone.querySelector(NO_TRIGGER_AUTOSAVE_SELECTOR)
     ? clonePreventingOnclone(clone)
     : null;
+  if (dirtyClone) carryProvenance(clone, dirtyClone);
 
   // Save clone: run hooks (freeze restore lives here), THEN strip [no-save]/[save-remove]
   // LAST (snapshot-algorithm step 7) so freeze-restored [no-save] content can't leak to disk.
@@ -656,7 +711,6 @@ export function captureForSaveAndComparison({ emitForSync = true, flushUndo = tr
     el.remove();
   }
   stripDiagnosticBeacons(clone);
-  const forSave = serializeSaveClone(clone, context);
 
   // Compare clone: strip every autosave-off region, then run hooks
   for (const el of compareClone.querySelectorAll(STRIP_FROM_COMPARISON)) {
@@ -667,10 +721,8 @@ export function captureForSaveAndComparison({ emitForSync = true, flushUndo = tr
     hook(compareClone);
   }
   stripSnapshotRegions(compareClone);
-  const forComparison = serializeForCompare(compareClone);
 
   // Dirty clone: same shape as the compare clone, one selector weaker.
-  let forDirty = forComparison;
   if (dirtyClone) {
     for (const el of dirtyClone.querySelectorAll(STRIP_FROM_DIRTY_CHECK)) {
       el.remove();
@@ -680,14 +732,88 @@ export function captureForSaveAndComparison({ emitForSync = true, flushUndo = tr
       hook(dirtyClone);
     }
     stripSnapshotRegions(dirtyClone);
-    forDirty = serializeForCompare(dirtyClone);
   }
 
-  const result = { forSave, forComparison, forDirty, captureId };
-  if (context && context.artifact) captureArtifacts.set(result, context.artifact);
+  return {
+    captureId,
+    context,
+    pageFormatMarker,
+    clone,
+    compareClone,
+    dirtyClone,
+    renderer: saveRenderer,
+    rendererSteps: saveRendererSteps,
+    pageStateReader: capturePageStateReader,
+    result: null,
+  };
+}
+
+/**
+ * Copy the page state this capture's bytes carry, before anything edits the clone.
+ *
+ * A capture taken for a request records the exact value its page tag carried, so the save
+ * the host accepts reads the state these bytes describe rather than the store's value at
+ * the moment the host answered.
+ */
+export function readCapturePageState(pin) {
+  if (!pin.context || !pin.pageStateReader) return;
+  let state = null;
+  try { state = pin.pageStateReader(pin.clone); } catch (_) {}
+  if (state) capturePageStates.set(pin.captureId, state);
+}
+
+/**
+ * Finish a save capture: the result object, and the artifact its renderer attached.
+ *
+ * @returns {{ forSave: string, forComparison: string, forDirty: string, captureId: number }}
+ */
+export function completeSaveCapture(pin, { forSave, forComparison, forDirty }) {
+  const result = { forSave, forComparison, forDirty, captureId: pin.captureId };
+  if (pin.context && pin.context.artifact) captureArtifacts.set(result, pin.context.artifact);
+  pin.result = result;
   return result;
 }
 
+/** Give back everything a capture holds. Idempotent. */
+export function releaseSaveCapture(pin) {
+  releaseCaptureArtifact(pin.result);
+  capturePageStates.delete(pin.captureId);
+  if (pin.context) pin.context.artifact = null;
+  pin.clone = pin.compareClone = pin.dirtyClone = null;
+  pin.result = null;
+}
+
+/**
+ * Single-capture function for both saving and comparison.
+ *
+ * Clones the DOM once, then clones that clone for comparison.
+ * More efficient than calling captureForSave() and captureForComparison() separately.
+ *
+ * The one-task form of a save capture: the same preparation, the same bytes and the same
+ * release as a job drained in the caller's own task, written out rather than resumed so
+ * nothing a page with small documents never uses has to be loaded to save it.
+ *
+ * @param {Object} options
+ * @param {boolean} options.emitForSync - Whether to emit snapshot-ready event (default: true)
+ * @param {boolean} options.forRequest - These bytes go to a host, so the renderer may
+ *   attach the verified model they came from, for the accepted save to adopt
+ * @returns {{ forSave: string, forComparison: string, forDirty: string, captureId: number }}
+ */
+export function captureForSaveAndComparison(opts = {}) {
+  const pin = prepareSaveCapture(opts);
+  try {
+    readCapturePageState(pin);
+    const today = serializeClone(pin.clone);
+    const forComparison = pin.compareClone ? serializeForCompare(pin.compareClone) : null;
+    const forDirty = pin.dirtyClone ? serializeForCompare(pin.dirtyClone) : forComparison;
+    const forSave = pin.renderer ? runSaveRenderer(pin.context, () => pin.renderer(pin.clone, today)) : today;
+    return completeSaveCapture(pin, { forSave, forComparison, forDirty });
+  } catch (err) {
+    releaseSaveCapture(pin);
+    throw err;
+  }
+}
+
 /**
  * Capture for a protected live-sync merge: the save-domain and loss-domain
  * clones of ONE snapshot, plus a WeakMap pairing every loss-domain element to
@@ -822,6 +948,11 @@ export function serializeForSync(clone) {
   return "<!DOCTYPE html>" + shell.slice(0, shell.length - endTag.length) + clone.innerHTML + endTag;
 }
 
+/** A private deep copy of a capture's clone, for work that outlives the capture event. */
+export function copyCaptureClone(clone) {
+  return clonePreventingOnclone(clone);
+}
+
 /**
  * PHASE 1-2 (body only): For live-sync between admin users.
  *