browser-commander 0.18.0 → 0.19.0

package/src/traces/mutation-stream.js

@@ -0,0 +1,232 @@
+/**
+ * The continuous half of a trace (issue #93).
+ *
+ * A checkpoint is a photograph; this is the film between two of them. Keeping
+ * it here rather than inside the recorder is what lets it answer the three
+ * questions issue #93 said it could not: what happened while a new document was
+ * loading, what happened inside an iframe, and what happened to state the DOM
+ * never mentions.
+ *
+ * The in-page half lives in `page-capture.js` and is installed twice over: once
+ * into every document that already exists, and once as an init script so every
+ * document created from now on is observed before its own code runs.
+ */
+
+import { withDeadline } from './deadline.js';
+import { framesOf, installInitScript } from './init-script.js';
+import {
+  drainMutationsInPage,
+  installMutationRecorderInPage,
+  RECORDER_GLOBAL,
+  stopMutationRecorderInPage,
+} from './page-capture.js';
+import { REDACTED } from './redaction.js';
+import { TRACE_DROP_REASON, TRACE_EVENT } from './schema.js';
+
+/** In-page records kept before the recorder starts counting drops instead. */
+export const DEFAULT_MAX_QUEUED_MUTATIONS = 5000;
+
+/**
+ * Build the recorder's continuous stream.
+ *
+ * @param {Object} options - Collaborators the stream needs
+ * @param {Object} options.page - The page being traced
+ * @param {Object} options.bundle - Open trace bundle
+ * @param {Function} options.record - Writes one timeline record
+ * @param {Object} options.identity - Trace identity, see `identity.js`
+ * @param {Function} options.note - Reports something that could not be done
+ * @param {Object} options.domOptions - Resolved `dom` options
+ * @param {Object} options.privacyOptions - Resolved privacy options
+ * @param {Object} options.limits - Resolved limits
+ * @param {number} options.captureTimeoutMs - Budget for one drain
+ * @param {Function} options.evaluateInPage - Fallback for a page without frames
+ * @returns {Object} `{install, installPersistent, drain, stop}`
+ */
+export function createMutationStream(options) {
+  const {
+    page,
+    bundle,
+    record,
+    identity,
+    note,
+    domOptions,
+    privacyOptions,
+    limits,
+    captureTimeoutMs,
+    evaluateInPage,
+  } = options;
+
+  const enabled = Boolean(domOptions.mutations);
+
+  /** What the in-page recorder is built with, wherever it is installed. */
+  const recorderOptions = {
+    globalName: RECORDER_GLOBAL,
+    redactSelectors: privacyOptions.redactSelectors,
+    redacted: REDACTED,
+    maxQueued: limits.maxQueuedMutations ?? DEFAULT_MAX_QUEUED_MUTATIONS,
+    liveState: domOptions.liveState !== false,
+  };
+
+  /**
+   * Run something in every frame of the page.
+   *
+   * An iframe is a separate document with a separate observer, so a trace that
+   * only ever spoke to the main frame recorded nothing that happened inside
+   * one. A frame that goes away mid-call is expected rather than a failure: a
+   * frame that has already gone has nothing left to report.
+   *
+   * @param {Function} fn - The function to evaluate
+   * @param {*} argument - Its argument
+   * @returns {Promise<Array>} One result per frame that answered
+   */
+  async function evaluateInFrames(fn, argument) {
+    const frames = framesOf(page);
+    if (frames.length === 0) {
+      return [await evaluateInPage(fn, argument)];
+    }
+    const results = await Promise.all(
+      frames.map(async (frame) => {
+        try {
+          return await frame.evaluate(fn, argument);
+        } catch {
+          return null;
+        }
+      })
+    );
+    return results.filter((result) => result !== null);
+  }
+
+  async function dropped(
+    member,
+    error,
+    reason = TRACE_DROP_REASON.CAPTURE_FAILED
+  ) {
+    await bundle.drop({
+      reason,
+      member,
+      detail: error.message ?? String(error),
+    });
+  }
+
+  /**
+   * Install the recorder into the documents that already exist.
+   *
+   * New documents are covered by the init script; this covers the one the trace
+   * started in, and any frame that was already open when it did.
+   *
+   * @returns {Promise<void>} Resolves once every frame has been asked
+   */
+  async function install() {
+    if (!enabled) {
+      return;
+    }
+    try {
+      await evaluateInFrames(installMutationRecorderInPage, recorderOptions);
+    } catch (error) {
+      await dropped('mutation-recorder', error);
+    }
+  }
+
+  /**
+   * Register the recorder to run before the code of every future document.
+   *
+   * This is what closes the gap between a navigation and the next checkpoint:
+   * without it a new document's own scripts run unobserved, and everything they
+   * build looks, on replay, like it had been there all along.
+   *
+   * @returns {Promise<Function|null>} Detach, when the engine offers one
+   */
+  async function installPersistent() {
+    if (!enabled) {
+      return null;
+    }
+    try {
+      return await installInitScript({
+        page,
+        fn: installMutationRecorderInPage,
+        arg: recorderOptions,
+        note,
+      });
+    } catch (error) {
+      await dropped('mutation-recorder-init', error);
+      return null;
+    }
+  }
+
+  /**
+   * Take everything every frame has queued and write it as one interval.
+   *
+   * @param {number} index - Checkpoint the interval ends at
+   * @returns {Promise<string|null>} The member written, when there was anything
+   */
+  async function drain(index) {
+    if (!enabled) {
+      return null;
+    }
+    try {
+      const drained = await withDeadline(
+        evaluateInFrames(drainMutationsInPage, RECORDER_GLOBAL),
+        captureTimeoutMs,
+        'trace mutation drain'
+      );
+
+      const batches = [];
+      let over = 0;
+      for (const frame of drained) {
+        over += frame?.dropped ?? 0;
+        for (const batch of frame?.batches ?? []) {
+          // The identity of the trace comes from here and the identity of the
+          // document comes from inside it: a frame handle is not something an
+          // NDJSON record can hold, so each document names itself.
+          batches.push({ ...identity.owner(), ...batch });
+        }
+      }
+
+      if (over) {
+        await dropped(
+          'mutations',
+          new Error(`${over} records over the in-page queue limit`),
+          TRACE_DROP_REASON.SIZE_LIMIT
+        );
+      }
+
+      // Frames are drained in parallel, so the batches come back grouped by
+      // frame; one ordered file is what a replay reads.
+      batches.sort((left, right) => (left.at ?? 0) - (right.at ?? 0));
+
+      const member = await bundle.writeMutations(index, batches);
+      if (member) {
+        await record(TRACE_EVENT.MUTATIONS, {
+          member,
+          batches: batches.length,
+          checkpoint: index,
+          frames: drained.length,
+        });
+      }
+      return member;
+    } catch (error) {
+      await dropped('mutations', error);
+      return null;
+    }
+  }
+
+  /**
+   * Switch the recorder off in every document that has one.
+   *
+   * A recorder left observing would keep queueing into a queue nobody drains.
+   *
+   * @returns {Promise<void>} Resolves once every frame has been asked
+   */
+  async function stop() {
+    if (!enabled) {
+      return;
+    }
+    try {
+      await evaluateInFrames(stopMutationRecorderInPage, RECORDER_GLOBAL);
+    } catch (error) {
+      note?.(`could not stop the in-page recorder: ${error.message}`);
+    }
+  }
+
+  return { install, installPersistent, drain, stop };
+}

package/src/traces/observers.js

@@ -51,17 +51,20 @@ export function interactionTarget(argument) {
 /**
  * Subscribe to everything a running trace listens to.
  *
- * @param {Object} options - `{commander, page, eventSources, record, note}`
+ * @param {Object} options - `{commander, page, eventSources, record, note, identity}`
  * @param {Object} [options.commander] - The commander being traced, when there
  *   is one; interactions and managed downloads are reached through it
  * @param {Object} options.page - The page being traced
  * @param {string[]} options.eventSources - Sources the caller asked for
  * @param {Function} options.record - Writes one timeline record
  * @param {Function} options.note - Reports an observer that could not attach
+ * @param {Object} [options.identity] - Trace identity, see `identity.js`; the
+ *   navigation observer is what moves it on to the next navigation, and the
+ *   interaction observer is what names an action (issue #93)
  * @returns {Function[]} One detach function per attached observer
  */
 export function attachTimelineObservers(options) {
-  const { commander, page, eventSources, record, note } = options;
+  const { commander, page, eventSources, record, note, identity } = options;
   const detachers = [];
 
   function subscribe(source, attach) {
@@ -90,8 +93,19 @@ export function attachTimelineObservers(options) {
     if (!manager) {
       // Without the manager the engine's own event still gives ordering.
       return onPage('framenavigated', (frame) => {
+        const isMain =
+          typeof frame?.parentFrame === 'function'
+            ? frame.parentFrame() === null
+            : true;
+        // A record's navigation is the one it happened during, so the counter
+        // moves on before the record that reports the move is written.
+        const navigationId = isMain
+          ? identity?.navigated()
+          : identity?.navigationId;
         void record(TRACE_EVENT.NAVIGATION, {
           phase: 'framenavigated',
+          navigationId,
+          mainFrame: isMain,
           url: typeof frame?.url === 'function' ? frame.url() : String(frame),
         });
       });
@@ -99,7 +113,14 @@ export function attachTimelineObservers(options) {
     const started = (info) =>
       void record(TRACE_EVENT.NAVIGATION, { phase: 'start', ...info });
     const completed = (info) =>
-      void record(TRACE_EVENT.NAVIGATION, { phase: 'complete', ...info });
+      void record(TRACE_EVENT.NAVIGATION, {
+        phase: 'complete',
+        // Everything recorded after this belongs to the document that just
+        // loaded, which is the distinction a replay needs to stop mixing two
+        // pages' changes together (issue #93).
+        navigationId: identity?.navigated(),
+        ...info,
+      });
     const changed = (info) =>
       void record(TRACE_EVENT.NAVIGATION, { phase: 'urlchange', ...info });
     manager.on('onNavigationStart', started);
@@ -215,9 +236,13 @@ export function attachTimelineObservers(options) {
       commander[name] = async (...args) => {
         const startedMs = Date.now();
         const target = interactionTarget(args[0]);
+        // Named before the call, so everything the action causes can be traced
+        // back to it even when the action itself ends in an exception.
+        const actionId = identity?.nextActionId() ?? null;
         try {
           const result = await original(...args);
           await record(TRACE_EVENT.INTERACTION, {
+            actionId,
             action: name,
             target,
             durationMs: Date.now() - startedMs,
@@ -226,6 +251,7 @@ export function attachTimelineObservers(options) {
           return result;
         } catch (error) {
           await record(TRACE_EVENT.INTERACTION, {
+            actionId,
             action: name,
             target,
             durationMs: Date.now() - startedMs,

package/src/traces/page-capture.js

@@ -201,12 +201,22 @@ export function captureSnapshotInPage(options) {
 }
 
 /**
- * Install the mutation recorder in the page.
+ * Install the in-page recorder.
  *
- * Reinstalling after a navigation is safe and expected: the queue lives on
- * the document, which a navigation replaces.
+ * Reinstalling after a navigation is safe and expected: the queue lives on the
+ * document, which a navigation replaces. Issue #93 is why this also runs as an
+ * init script - installing it only from `checkpoint()` left the interval
+ * between a navigation and the next checkpoint unrecorded, so a page that
+ * built its first screen during `DOMContentLoaded` replayed as if it had
+ * always been there.
  *
- * @param {Object} options - `{globalName, redactSelectors, redacted, maxQueued}`
+ * Two kinds of change are recorded. DOM mutations come from a
+ * `MutationObserver`, with the position information a deterministic replay
+ * needs. Live control state - what was typed, checked, selected, focused and
+ * scrolled - produces no mutation records at all, so it is observed from the
+ * events that carry it and recorded on the same queue, in the same order.
+ *
+ * @param {Object} options - `{globalName, redactSelectors, redacted, maxQueued, liveState}`
  * @returns {boolean} True when a recorder is now observing
  */
 export function installMutationRecorderInPage(options) {
@@ -215,6 +225,7 @@ export function installMutationRecorderInPage(options) {
     redactSelectors = [],
     redacted = '[redacted]',
     maxQueued = 5000,
+    liveState = true,
   } = options || {};
 
   if (window[globalName] && window[globalName].observing) {
@@ -226,6 +237,11 @@ export function installMutationRecorderInPage(options) {
     queue: [],
     dropped: 0,
     sequence: 0,
+    // Every document gets its own name, so a batch drained from an iframe is
+    // never mistaken for one drained from the page that holds it.
+    frameId: `frame-${Math.random().toString(36).slice(2, 10)}`,
+    frameName: window.name || null,
+    mainFrame: window.top === window,
   };
 
   const isSecret = (node) => {
@@ -264,6 +280,13 @@ export function installMutationRecorderInPage(options) {
     return steps.join(' > ');
   };
 
+  const childIndexOf = (parent, node) => {
+    if (!parent || !node || !parent.childNodes) {
+      return -1;
+    }
+    return Array.prototype.indexOf.call(parent.childNodes, node);
+  };
+
   const describe = (node) => {
     if (!node) {
       return null;
@@ -287,15 +310,60 @@ export function installMutationRecorderInPage(options) {
     };
   };
 
-  const observer = new MutationObserver((records) => {
+  /**
+   * Describe a node that is no longer where it was.
+   *
+   * A removed node has no `parentElement`, so `pathOf` can only name the node
+   * itself. Its position in the parent it left is what a replay needs, and
+   * that is read from the siblings the record still points at.
+   *
+   * @param {Node} node - The node that moved or was removed
+   * @param {MutationRecord} record - The record it came from
+   * @returns {Object|null} The description, with `index` when one is knowable
+   */
+  const describeAt = (node, record) => {
+    const described = describe(node);
+    if (!described) {
+      return null;
+    }
+    const parent = record.target;
+    const own = childIndexOf(parent, node);
+    if (own >= 0) {
+      described.index = own;
+      return described;
+    }
+    const after = childIndexOf(parent, record.previousSibling);
+    if (after >= 0) {
+      described.index = after + 1;
+      return described;
+    }
+    const before = childIndexOf(parent, record.nextSibling);
+    described.index = before >= 0 ? before : null;
+    return described;
+  };
+
+  const push = (batch) => {
     if (state.queue.length >= maxQueued) {
-      state.dropped += records.length;
-      return;
+      state.dropped += 1;
+      return false;
     }
-    const batch = {
+    state.queue.push({
       sequence: ++state.sequence,
       at: Date.now(),
       url: location.href,
+      frameId: state.frameId,
+      mainFrame: state.mainFrame,
+      ...batch,
+    });
+    return true;
+  };
+
+  const observer = new MutationObserver((records) => {
+    if (state.queue.length >= maxQueued) {
+      state.dropped += records.length;
+      return;
+    }
+    push({
       records: records.map((record) => {
         const entry = {
           kind: record.type,
@@ -317,13 +385,21 @@ export function installMutationRecorderInPage(options) {
             ? redacted
             : record.target.nodeValue;
         } else {
-          entry.added = [...record.addedNodes].map(describe);
-          entry.removed = [...record.removedNodes].map(describe);
+          // Where a node went is as much of the change as what it was:
+          // appending every addition to the end replays an insertion before a
+          // sibling in the wrong place, and drops moves and removals entirely.
+          entry.added = [...record.addedNodes].map((node) =>
+            describeAt(node, record)
+          );
+          entry.removed = [...record.removedNodes].map((node) =>
+            describeAt(node, record)
+          );
+          entry.previous = describe(record.previousSibling);
+          entry.next = describe(record.nextSibling);
         }
         return entry;
       }),
-    };
-    state.queue.push(batch);
+    });
   });
 
   observer.observe(document, {
@@ -335,8 +411,139 @@ export function installMutationRecorderInPage(options) {
     characterDataOldValue: true,
   });
 
+  const liveDetachers = [];
+  if (liveState) {
+    const lastValues = new WeakMap();
+    const lastLive = new WeakMap();
+
+    // One interaction fires more than one event for the same state: typing
+    // into a field and then leaving it fires `input` for each keystroke and
+    // `change` for the value it ended at, and a checkbox fires both for the
+    // one click. Replaying a state that is already on screen shows nothing, so
+    // a repeat of what this element last reported is not recorded.
+    const repeats = (element, property, after) => {
+      const seen = lastLive.get(element) || {};
+      const asText = JSON.stringify(after === undefined ? null : after);
+      if (seen[property] === asText) {
+        return true;
+      }
+      seen[property] = asText;
+      lastLive.set(element, seen);
+      return false;
+    };
+
+    const liveEntry = (target, property, after, before) => ({
+      kind: 'live-state',
+      property,
+      target: describe(target),
+      before: before === undefined ? null : before,
+      after,
+    });
+
+    const pushLive = (entry) => {
+      // Scrolling fires far more often than it changes anything worth
+      // replaying, so consecutive scrolls of one element collapse into the
+      // position it ended at. Every other property keeps each step, because
+      // "stepwise replay must show each state" (issue #93).
+      const last = state.queue[state.queue.length - 1];
+      const only = last && last.records.length === 1 ? last.records[0] : null;
+      if (
+        entry.property === 'scroll' &&
+        only &&
+        only.kind === 'live-state' &&
+        only.property === 'scroll' &&
+        only.target?.path === entry.target?.path
+      ) {
+        only.after = entry.after;
+        last.at = Date.now();
+        return;
+      }
+      push({ records: [entry] });
+    };
+
+    const valueOf = (element) =>
+      isSecret(element)
+        ? redacted
+        : element.isContentEditable
+          ? element.textContent
+          : element.value;
+
+    const onValue = (event) => {
+      const element = event.target;
+      if (!element || element.nodeType !== 1) {
+        return;
+      }
+      if (element.type === 'checkbox' || element.type === 'radio') {
+        if (!repeats(element, 'checked', element.checked)) {
+          pushLive(liveEntry(element, 'checked', element.checked));
+        }
+        return;
+      }
+      if (element.localName === 'select') {
+        const chosen = [...element.selectedOptions].map((option) =>
+          isSecret(element) ? redacted : option.value
+        );
+        if (!repeats(element, 'selected', chosen)) {
+          pushLive(liveEntry(element, 'selected', chosen));
+        }
+        return;
+      }
+      if (
+        element.localName !== 'input' &&
+        element.localName !== 'textarea' &&
+        !element.isContentEditable
+      ) {
+        return;
+      }
+      const after = valueOf(element);
+      if (repeats(element, 'value', after)) {
+        return;
+      }
+      const before = lastValues.get(element);
+      lastValues.set(element, after);
+      pushLive(liveEntry(element, 'value', after, before));
+    };
+
+    const onFocus = (event) => {
+      pushLive(liveEntry(event.target, 'focus', event.type === 'focusin'));
+    };
+
+    const onScroll = (event) => {
+      const target = event.target;
+      const scrolling =
+        target === document || target === document.documentElement || !target
+          ? null
+          : target;
+      pushLive({
+        kind: 'live-state',
+        property: 'scroll',
+        target: scrolling ? describe(scrolling) : null,
+        before: null,
+        after: scrolling
+          ? { top: scrolling.scrollTop, left: scrolling.scrollLeft }
+          : { top: window.scrollY, left: window.scrollX },
+      });
+    };
+
+    const listen = (type, handler) => {
+      document.addEventListener(type, handler, true);
+      liveDetachers.push(() =>
+        document.removeEventListener(type, handler, true)
+      );
+    };
+
+    listen('input', onValue);
+    listen('change', onValue);
+    listen('focusin', onFocus);
+    listen('focusout', onFocus);
+    listen('scroll', onScroll);
+  }
+
   state.stop = () => {
     observer.disconnect();
+    for (const detach of liveDetachers) {
+      detach();
+    }
     state.observing = false;
   };
 
@@ -348,18 +555,47 @@ export function installMutationRecorderInPage(options) {
  * Take everything the recorder has queued and leave the queue empty.
  *
  * @param {string} globalName - Global the recorder lives on
- * @returns {Object} `{batches, dropped}`
+ * @returns {Object} `{batches, dropped, installed, frameId, mainFrame}`
  */
 export function drainMutationsInPage(globalName) {
   const state = window[globalName || '__browserCommanderTrace__'];
   if (!state) {
-    return { batches: [], dropped: 0, installed: false };
+    return {
+      batches: [],
+      dropped: 0,
+      installed: false,
+      frameId: null,
+      mainFrame: window.top === window,
+      url: location.href,
+    };
   }
   const batches = state.queue;
   const dropped = state.dropped;
   state.queue = [];
   state.dropped = 0;
-  return { batches, dropped, installed: true };
+  return {
+    batches,
+    dropped,
+    installed: true,
+    frameId: state.frameId,
+    mainFrame: state.mainFrame,
+    url: location.href,
+  };
+}
+
+/**
+ * Stop the in-page recorder without waiting for the document to go away.
+ *
+ * @param {string} globalName - Global the recorder lives on
+ * @returns {boolean} True when a recorder was stopped
+ */
+export function stopMutationRecorderInPage(globalName) {
+  const state = window[globalName || '__browserCommanderTrace__'];
+  if (!state || !state.observing) {
+    return false;
+  }
+  state.stop();
+  return true;
 }
 
 /* c8 ignore stop */