browser-commander 0.18.0 → 0.19.0

package/src/traces/recorder.js

@@ -7,13 +7,12 @@
  * request, so every caller does not have to.
  */
 
-import {
-  captureSnapshotInPage,
-  drainMutationsInPage,
-  installMutationRecorderInPage,
-  RECORDER_GLOBAL,
-} from './page-capture.js';
+import { captureSnapshotInPage } from './page-capture.js';
 import { openTraceBundle } from './bundle.js';
+import { openTraceLinks } from './links.js';
+import { createTraceIdentity } from './identity.js';
+import { withDeadline } from './deadline.js';
+import { createMutationStream } from './mutation-stream.js';
 import { attachTimelineObservers } from './observers.js';
 import {
   normalizePrivacyOptions,
@@ -24,17 +23,25 @@ import {
 import {
   createManifest,
   DEFAULT_CAPTURE_TIMEOUT,
+  TRACE_CHECKPOINT_REASON,
   TRACE_DROP_REASON,
   TRACE_EVENT,
   TRACE_EVENT_SOURCES,
   TRACE_MODE,
   TRACE_OUTCOME,
+  TRACE_SCHEMA_VERSION,
 } from './schema.js';
 
 /** DOM capture defaults: what a checkpoint holds unless the caller narrows it. */
 const DEFAULT_DOM_OPTIONS = Object.freeze({
   html: true,
   liveControlState: true,
+  /**
+   * Record what typing, checking, selecting, focus and scroll did, as it
+   * happens (issue #93). Reading it only at checkpoints meant a replay could
+   * show an empty field a moment after the user finished filling it in.
+   */
+  liveState: true,
   mutations: false,
   openShadowRoots: true,
 });
@@ -67,36 +74,6 @@ function normalizeEvents(events) {
   return [...new Set(events)];
 }
 
-/**
- * Run a capture with its own deadline.
- *
- * A page that stopped answering must cost the trace one record, not the run.
- *
- * @param {Promise} work - The capture
- * @param {number} timeoutMs - Budget
- * @param {string} what - Name used in the timeout message
- * @returns {Promise<*>} The capture's result
- */
-async function withDeadline(work, timeoutMs, what) {
-  if (!timeoutMs) {
-    return work;
-  }
-  let timer;
-  try {
-    return await Promise.race([
-      work,
-      new Promise((_resolve, reject) => {
-        timer = setTimeout(
-          () => reject(new Error(`${what} timed out after ${timeoutMs}ms`)),
-          timeoutMs
-        );
-      }),
-    ]);
-  } finally {
-    clearTimeout(timer);
-  }
-}
-
 /**
  * Start recording a session.
  *
@@ -109,11 +86,13 @@ export async function startTrace(options = {}) {
     page = commander?.page,
     output,
     mode = TRACE_MODE.CHECKPOINTS,
+    initialCheckpoint,
     screenshots = 'checkpoints',
     dom = {},
     events,
     privacy = {},
     limits = {},
+    links = null,
     strict = false,
     captureTimeoutMs = DEFAULT_CAPTURE_TIMEOUT,
     commanderVersion = null,
@@ -133,10 +112,48 @@ export async function startTrace(options = {}) {
   const eventSources = normalizeEvents(events);
   const privacyOptions = normalizePrivacyOptions(privacy);
   const engine = commander?.engine ?? null;
-
-  const bundle = await openTraceBundle({ output, limits, strict, now });
+  const identity = createTraceIdentity({ page });
+
+  // A continuous trace records changes, and a change is only meaningful against
+  // something. Taking the base snapshot automatically is the safe default,
+  // because a trace that starts mid-air replays as a blank page (issue #93); a
+  // checkpoints-only trace is a list of moments the caller named, so nothing is
+  // added to it uninvited. Passing a string names the base checkpoint.
+  const baseCheckpoint =
+    initialCheckpoint ?? recorderMode === TRACE_MODE.CONTINUOUS;
+  const baseCheckpointName =
+    typeof baseCheckpoint === 'string' ? baseCheckpoint : 'initial';
+
+  // The Links Notation export is a second view of the same records, written
+  // as they are recorded (issue #94), so a run that is killed leaves a
+  // readable export of everything that had happened. The bundle stays
+  // authoritative: nothing is written here that is not written there first.
+  let linksSink = null;
+  const bundle = await openTraceBundle({
+    output,
+    limits,
+    strict,
+    now,
+    onEvent: (event) => linksSink?.event(event),
+  });
   const startedAt = new Date(now()).toISOString();
 
+  if (links) {
+    if (typeof links.output !== 'string' || links.output === '') {
+      throw new Error('trace links require an output path');
+    }
+    linksSink = await openTraceLinks({
+      output: links.output,
+      include: links.include,
+      bundlePath: bundle.root,
+      schemaVersion: TRACE_SCHEMA_VERSION,
+      mode: recorderMode,
+      engine,
+      startedAt,
+      commanderVersion,
+    });
+  }
+
   let stopped = false;
   let checkpointIndex = 0;
   const checkpoints = [];
@@ -156,6 +173,9 @@ export async function startTrace(options = {}) {
     }
     return await bundle.appendEvent({
       kind,
+      // Who this happened to comes first so a payload that knows better - a
+      // drain that names the frame it came from, say - can say so.
+      ...identity.owner(),
       ...redactValue(payload, privacyOptions),
     });
   }
@@ -169,61 +189,18 @@ export async function startTrace(options = {}) {
     return await page.evaluate(fn, argument);
   }
 
-  async function installMutationRecorder() {
-    if (!domOptions.mutations) {
-      return;
-    }
-    try {
-      await evaluateInPage(installMutationRecorderInPage, {
-        globalName: RECORDER_GLOBAL,
-        redactSelectors: privacyOptions.redactSelectors,
-        redacted: REDACTED,
-        maxQueued: limits.maxQueuedMutations ?? 5000,
-      });
-    } catch (error) {
-      await bundle.drop({
-        reason: TRACE_DROP_REASON.CAPTURE_FAILED,
-        member: 'mutation-recorder',
-        detail: error.message,
-      });
-    }
-  }
-
-  async function drainMutations(index) {
-    if (!domOptions.mutations) {
-      return null;
-    }
-    try {
-      const drained = await withDeadline(
-        evaluateInPage(drainMutationsInPage, RECORDER_GLOBAL),
-        captureTimeoutMs,
-        'trace mutation drain'
-      );
-      if (drained?.dropped) {
-        await bundle.drop({
-          reason: TRACE_DROP_REASON.SIZE_LIMIT,
-          member: 'mutations',
-          detail: `${drained.dropped} records over the in-page queue limit`,
-        });
-      }
-      const member = await bundle.writeMutations(index, drained?.batches ?? []);
-      if (member) {
-        await record(TRACE_EVENT.MUTATIONS, {
-          member,
-          batches: drained.batches.length,
-          checkpoint: index,
-        });
-      }
-      return member;
-    } catch (error) {
-      await bundle.drop({
-        reason: TRACE_DROP_REASON.CAPTURE_FAILED,
-        member: 'mutations',
-        detail: error.message,
-      });
-      return null;
-    }
-  }
+  const mutations = createMutationStream({
+    page,
+    bundle,
+    record,
+    identity,
+    note,
+    domOptions,
+    privacyOptions,
+    limits,
+    captureTimeoutMs,
+    evaluateInPage,
+  });
 
   async function screenshot(reason) {
     const wanted =
@@ -267,7 +244,7 @@ export async function startTrace(options = {}) {
 
     // Mutations are drained first so the batches belong to the interval that
     // ended here, not to the one that starts now.
-    await drainMutations(index - 1 > 0 ? index - 1 : 0);
+    await mutations.drain(index - 1 > 0 ? index - 1 : 0);
 
     let captured = null;
     try {
@@ -318,9 +295,10 @@ export async function startTrace(options = {}) {
     checkpoints.push(entry);
     await record(TRACE_EVENT.CHECKPOINT, entry);
 
-    // A navigation replaces the document and with it the observer, so the
-    // recorder is reinstalled at every checkpoint rather than only at start.
-    await installMutationRecorder();
+    // The init script covers every document created from here on; this covers
+    // a frame that was attached without one, which is cheap because installing
+    // over a recorder that is already observing does nothing.
+    await mutations.install();
 
     return entry;
   }
@@ -331,15 +309,31 @@ export async function startTrace(options = {}) {
     eventSources,
     record,
     note,
+    identity,
   });
 
+  // Registered before the first record, so a navigation that starts in the same
+  // tick as the trace does is still recorded from its first mutation.
+  const detachInitScript = await mutations.installPersistent();
+  if (detachInitScript) {
+    detachers.push(detachInitScript);
+  }
+
   await record(TRACE_EVENT.TRACE_START, {
     mode: recorderMode,
     engine,
     dom: domOptions,
     events: eventSources,
+    initialCheckpoint: Boolean(baseCheckpoint),
   });
-  await installMutationRecorder();
+  await mutations.install();
+
+  if (baseCheckpoint) {
+    await checkpoint(baseCheckpointName, {
+      actor: 'recorder',
+      reason: TRACE_CHECKPOINT_REASON.INITIAL,
+    });
+  }
 
   /**
    * Stop recording and write the manifest.
@@ -353,7 +347,7 @@ export async function startTrace(options = {}) {
     }
     const { discard = false, error = null } = stopOptions;
 
-    await drainMutations(checkpointIndex);
+    await mutations.drain(checkpointIndex);
     if (error) {
       await record(TRACE_EVENT.PAGE_ERROR, {
         message: error.message ?? String(error),
@@ -364,6 +358,10 @@ export async function startTrace(options = {}) {
     await record(TRACE_EVENT.TRACE_STOP, { discarded: discard });
     stopped = true;
 
+    // The documents that exist stop observing here; the engine's init-script
+    // registration is removed with the detachers below.
+    await mutations.stop();
+
     for (const detach of detachers.reverse()) {
       try {
         await detach();
@@ -382,6 +380,14 @@ export async function startTrace(options = {}) {
         engine,
         dom: domOptions,
         events: eventSources,
+        replay: {
+          checkpoints: true,
+          mutations: Boolean(domOptions.mutations),
+          childListPositions: Boolean(domOptions.mutations),
+          liveState:
+            Boolean(domOptions.mutations) && domOptions.liveState !== false,
+          identifiers: true,
+        },
         privacy: {
           redactSelectors: privacyOptions.redactSelectors,
           redactAttributes: privacyOptions.redactAttributes,
@@ -392,17 +398,27 @@ export async function startTrace(options = {}) {
       })
     );
 
+    // Closed after the manifest, because the closing link reports the outcome
+    // the manifest settled on, and the control diffs are read back out of the
+    // finished bundle rather than kept in memory for the length of a run.
+    if (linksSink) {
+      await linksSink.close({ manifest, bundlePath: bundle.root });
+    }
+
     const result = {
       path: bundle.root,
       manifest,
       checkpoints: [...checkpoints],
-      problems: [...bundle.problems],
+      problems: [...bundle.problems, ...(linksSink?.problems ?? [])],
+      links: linksSink?.path ?? null,
     };
     stopped = result;
 
     if (discard) {
       const fs = await import('node:fs/promises');
       await fs.rm(bundle.root, { recursive: true, force: true });
+      // An export of a bundle that no longer exists points at nothing.
+      await linksSink?.discard();
       result.discarded = true;
     }
 
@@ -411,6 +427,7 @@ export async function startTrace(options = {}) {
 
   return {
     path: bundle.root,
+    links: linksSink?.path ?? null,
     mode: recorderMode,
     checkpoint,
     /**

package/src/traces/schema.js

@@ -11,8 +11,15 @@
  *
  * Readers refuse a major version they were not written for rather than
  * guessing at the meaning of unknown records.
+ *
+ * Version 2 (issue #93) is what makes continuous replay complete: mutation
+ * batches name the frame they came from, child-list records carry the position
+ * a node was inserted at or removed from, and live control state - typing,
+ * checking, selecting, focus and scroll - is recorded as it happens instead of
+ * only at checkpoints. A version 1 reader given a version 2 bundle would
+ * quietly replay a different page, so it refuses instead.
  */
-export const TRACE_SCHEMA_VERSION = 1;
+export const TRACE_SCHEMA_VERSION = 2;
 
 /** Name every reader looks for inside a bundle. */
 export const TRACE_FILES = Object.freeze({
@@ -49,6 +56,40 @@ export const TRACE_EVENT = Object.freeze({
   DROPPED: 'dropped',
 });
 
+/** Record kinds inside a mutation batch. */
+export const TRACE_MUTATION_KIND = Object.freeze({
+  ATTRIBUTES: 'attributes',
+  CHARACTER_DATA: 'characterData',
+  CHILD_LIST: 'childList',
+  /**
+   * A change to what an element holds rather than to the document.
+   *
+   * Typing into an input, checking a box, choosing an option, moving focus and
+   * scrolling all change what the user sees and none of them mutate the DOM,
+   * so no observer reports them and replay skipped them entirely (issue #93).
+   */
+  LIVE_STATE: 'live-state',
+});
+
+/** What a `live-state` record changed. */
+export const TRACE_LIVE_STATE = Object.freeze({
+  VALUE: 'value',
+  CHECKED: 'checked',
+  SELECTED: 'selected',
+  FOCUS: 'focus',
+  SCROLL: 'scroll',
+});
+
+/** Why a checkpoint was taken. */
+export const TRACE_CHECKPOINT_REASON = Object.freeze({
+  /** The base snapshot every later record is a change against. */
+  INITIAL: 'initial',
+  /** A caller named this moment. */
+  CHECKPOINT: 'checkpoint',
+  /** The run failed here. */
+  FAILURE: 'failure',
+});
+
 /** Event families a caller can subscribe the recorder to. */
 export const TRACE_EVENT_SOURCES = Object.freeze([
   'navigation',
@@ -101,6 +142,27 @@ export function sequenceName(index) {
   return String(index).padStart(4, '0');
 }
 
+/**
+ * What a bundle's records let a viewer reproduce.
+ *
+ * Issue #93 asked for the viewer to stop claiming more than it can do. Saying
+ * it in the manifest is better than saying it in the viewer's markup: a bundle
+ * recorded by an older version is honestly described by its own manifest, and
+ * any of the three readers can tell a caller what they are looking at.
+ */
+export const DEFAULT_REPLAY_SUPPORT = Object.freeze({
+  /** Checkpoints are snapshots, always replayable. */
+  checkpoints: true,
+  /** DOM mutations are recorded between checkpoints. */
+  mutations: false,
+  /** Child-list records say where a node went, so removals and moves replay. */
+  childListPositions: false,
+  /** Typing, checking, selecting, focus and scroll are recorded as they happen. */
+  liveState: false,
+  /** Every record names the context, page, navigation and frame it belongs to. */
+  identifiers: false,
+});
+
 /**
  * Build the manifest of a trace.
  *
@@ -124,6 +186,7 @@ export function createManifest(options = {}) {
     dom = {},
     events = [],
     dropped = 0,
+    replay = {},
   } = options;
 
   return {
@@ -140,6 +203,7 @@ export function createManifest(options = {}) {
     runtime,
     events: [...events],
     dom: { ...dom },
+    replay: { ...DEFAULT_REPLAY_SUPPORT, ...replay },
     privacy: { ...privacy },
     limits: { ...limits },
     counts: { checkpoints: 0, events: 0, mutationBatches: 0, ...counts },

package/src/traces/viewer.js

@@ -43,6 +43,7 @@ const VIEWER_STYLE = `
   header { grid-column: 1 / -1; padding: 0.5rem 1rem; border-bottom: 1px solid #8884; }
   h1 { font-size: 1rem; margin: 0 0 0.25rem; }
   .meta { font-size: 0.75rem; opacity: 0.75; }
+  .meta.replay { opacity: 0.9; font-style: italic; }
   aside { overflow: auto; border-right: 1px solid #8884; }
   main { display: grid; grid-template-rows: auto 1fr auto; overflow: hidden; }
   ol { list-style: none; margin: 0; padding: 0; font-size: 0.8rem; }
@@ -87,19 +88,117 @@ function renderHtml(index, html) {
   frame.srcdoc = html == null ? '<p style="font:1rem system-ui">no HTML captured for this checkpoint</p>' : html;
 }
 
+function nodeOf(parsed, described) {
+  if (!described) return null;
+  if (described.type === 'text') {
+    return parsed.createTextNode(described.text == null ? '' : described.text);
+  }
+  if (described.type === 'element' && described.html) {
+    const holder = parsed.createElement('template');
+    holder.innerHTML = described.html;
+    return holder.content.firstChild;
+  }
+  return null;
+}
+
+function looksLike(node, described) {
+  if (!node || !described) return false;
+  if (described.type === 'text') return node.nodeType === 3;
+  if (described.type !== 'element') return false;
+  if (node.nodeType !== 1 || node.localName !== described.tag) return false;
+  return described.id ? node.id === described.id : true;
+}
+
+function childLike(parent, described) {
+  if (!described) return null;
+  // An id is identity; an index is only a hint. A node that moved is reported
+  // at the position it moved *to*, so the index alone would find the wrong one.
+  if (described.id) {
+    for (const child of parent.childNodes) {
+      if (child.nodeType === 1 && child.id === described.id) return child;
+    }
+  }
+  const at = described.index == null ? null : parent.childNodes[described.index];
+  if (looksLike(at, described)) return at;
+  for (const child of parent.childNodes) {
+    if (looksLike(child, described)) return child;
+  }
+  return null;
+}
+
+function insertAt(parent, node, index) {
+  if (!node) return false;
+  const reference = index == null ? null : parent.childNodes[index] || null;
+  parent.insertBefore(node, reference);
+  return true;
+}
+
+function applyLiveState(parsed, target, record) {
+  const value = record.after;
+  const text = value == null ? '' : String(value);
+  if (record.property === 'scroll') {
+    // A static copy of a document cannot be scrolled, so where it was
+    // scrolled to is written onto the element rather than quietly lost.
+    const element = target || parsed.documentElement;
+    const at = value || {};
+    element.setAttribute('data-bc-scroll', at.top + ',' + at.left);
+    return true;
+  }
+  if (!target) return false;
+  if (record.property === 'value') {
+    // Serialized HTML carries attributes, not properties, so what was typed
+    // is written where a re-parse will still find it.
+    if (target.localName === 'textarea' || target.hasAttribute('contenteditable')) {
+      target.textContent = text;
+    } else {
+      target.setAttribute('value', text);
+    }
+    return true;
+  }
+  if (record.property === 'checked') {
+    if (value) target.setAttribute('checked', '');
+    else target.removeAttribute('checked');
+    return true;
+  }
+  if (record.property === 'selected') {
+    const chosen = (Array.isArray(value) ? value : []).map(String);
+    for (const option of target.querySelectorAll('option')) {
+      if (chosen.indexOf(option.value) !== -1) option.setAttribute('selected', '');
+      else option.removeAttribute('selected');
+    }
+    return true;
+  }
+  if (record.property === 'focus') {
+    for (const had of parsed.querySelectorAll('[data-bc-focus]')) {
+      had.removeAttribute('data-bc-focus');
+    }
+    if (value) target.setAttribute('data-bc-focus', '');
+    return true;
+  }
+  return false;
+}
+
 function applyMutations(index, upTo) {
   const html = trace.html[index];
   if (html == null) return null;
   const parsed = new DOMParser().parseFromString(html, 'text/html');
   const batches = trace.mutations[index] || [];
   let applied = 0;
+  let skipped = 0;
   for (const batch of batches.slice(0, upTo)) {
+    // The stage holds one document. A batch from an iframe belongs to a
+    // different one, so it is counted rather than applied to the wrong page.
+    if (batch.mainFrame === false) { skipped += (batch.records || []).length; continue; }
     for (const record of batch.records) {
       const target = record.target && record.target.path
         ? parsed.querySelector(record.target.path)
         : null;
-      if (!target) continue;
-      if (record.kind === 'attributes' && record.attribute) {
+      if (record.kind === 'live-state') {
+        if (applyLiveState(parsed, target, record)) applied++; else skipped++;
+      } else if (!target) {
+        skipped++;
+        continue;
+      } else if (record.kind === 'attributes' && record.attribute) {
         if (record.after == null) target.removeAttribute(record.attribute);
         else target.setAttribute(record.attribute, record.after);
         applied++;
@@ -107,17 +206,32 @@ function applyMutations(index, upTo) {
         target.textContent = record.after == null ? '' : record.after;
         applied++;
       } else if (record.kind === 'childList') {
-        for (const added of record.added || []) {
-          if (added && added.type === 'element' && added.html) {
-            target.insertAdjacentHTML('beforeend', added.html);
-            applied++;
-          }
+        // Where a node went is as much of the change as what it was: appending
+        // every addition to the end replays an insertion before a sibling in
+        // the wrong place, and drops removals entirely (issue #93).
+        for (const removed of record.removed || []) {
+          const node = childLike(target, removed);
+          if (node) { target.removeChild(node); applied++; } else { skipped++; }
         }
+        const added = (record.added || []).filter(Boolean)
+          .slice().sort((a, b) => (a.index == null ? 1e9 : a.index) - (b.index == null ? 1e9 : b.index));
+        for (const entry of added) {
+          if (insertAt(target, nodeOf(parsed, entry), entry.index)) applied++;
+          else skipped++;
+        }
+      } else {
+        skipped++;
       }
-      if (target.style) target.style.outline = '2px solid #f59e0b';
+      if (target && target.style) target.style.outline = '2px solid #f59e0b';
     }
   }
-  return { html: '<!DOCTYPE html>' + parsed.documentElement.outerHTML, applied };
+  return { html: '<!DOCTYPE html>' + parsed.documentElement.outerHTML, applied, skipped };
+}
+
+function stepText(step, total, result) {
+  const base = 'batch ' + step + ' of ' + total;
+  if (!result || !result.skipped) return base;
+  return base + ' · ' + result.skipped + ' not replayable here';
 }
 
 function showCheckpoint(index) {
@@ -171,7 +285,7 @@ document.getElementById('play').addEventListener('click', async () => {
   for (let step = 1; step <= batches.length; step++) {
     const result = applyMutations(selectedCheckpoint, step);
     if (result) frame.srcdoc = result.html;
-    stepLabel.textContent = 'batch ' + step + ' of ' + batches.length;
+    stepLabel.textContent = stepText(step, batches.length, result);
     await new Promise((resolve) => setTimeout(resolve, 250));
   }
 });
@@ -182,7 +296,7 @@ document.getElementById('step-forward').addEventListener('click', () => {
   mutationStep = Math.min(mutationStep + 1, batches.length);
   const result = applyMutations(selectedCheckpoint, mutationStep);
   if (result) frame.srcdoc = result.html;
-  stepLabel.textContent = 'batch ' + mutationStep + ' of ' + batches.length;
+  stepLabel.textContent = stepText(mutationStep, batches.length, result);
 });
 
 document.getElementById('reset').addEventListener('click', () => {
@@ -192,6 +306,37 @@ document.getElementById('reset').addEventListener('click', () => {
 if (trace.checkpoints.length) showCheckpoint(trace.checkpoints[0].index);
 `;
 
+/**
+ * What this bundle can be replayed from, in the viewer's own words.
+ *
+ * Issue #93 asked for the viewer to say plainly that it reconstructs a run
+ * rather than replaying a recording of one, and to stop implying it can show
+ * things the bundle does not contain. A checkpoints-only trace has nothing
+ * between its checkpoints; a continuous one has as much as it recorded.
+ *
+ * @param {Object} manifest - The bundle's manifest
+ * @returns {string} A one-line description of what replay covers
+ */
+export function replaySummary(manifest) {
+  const replay = manifest?.replay ?? {};
+  const covered = [];
+  if (replay.mutations) {
+    covered.push('DOM mutations');
+  }
+  if (replay.childListPositions) {
+    covered.push('insertion positions and removals');
+  }
+  if (replay.liveState) {
+    covered.push('live control state');
+  }
+  if (replay.identifiers) {
+    covered.push('page and frame identities');
+  }
+  return covered.length === 0
+    ? 'partial diagnostic replay: checkpoints only, nothing between them'
+    : `partial diagnostic replay: checkpoints, ${covered.join(', ')}`;
+}
+
 /**
  * Collect everything the viewer embeds.
  *
@@ -266,6 +411,7 @@ export async function renderTraceViewer(reader, options = {}) {
     ${manifest.counts?.checkpoints ?? 0} checkpoints · ${manifest.counts?.events ?? 0} events ·
     ${manifest.dropped ?? 0} dropped · started ${manifest.startedAt ?? 'unknown'}
   </div>
+  <div class="meta replay">${replaySummary(manifest)}</div>
 </header>
 <aside><ol id="timeline"></ol></aside>
 <main>