browser-commander 0.17.2 → 0.19.0

package/src/traces/mutation-stream.js

@@ -0,0 +1,232 @@
+/**
+ * The continuous half of a trace (issue #93).
+ *
+ * A checkpoint is a photograph; this is the film between two of them. Keeping
+ * it here rather than inside the recorder is what lets it answer the three
+ * questions issue #93 said it could not: what happened while a new document was
+ * loading, what happened inside an iframe, and what happened to state the DOM
+ * never mentions.
+ *
+ * The in-page half lives in `page-capture.js` and is installed twice over: once
+ * into every document that already exists, and once as an init script so every
+ * document created from now on is observed before its own code runs.
+ */
+
+import { withDeadline } from './deadline.js';
+import { framesOf, installInitScript } from './init-script.js';
+import {
+  drainMutationsInPage,
+  installMutationRecorderInPage,
+  RECORDER_GLOBAL,
+  stopMutationRecorderInPage,
+} from './page-capture.js';
+import { REDACTED } from './redaction.js';
+import { TRACE_DROP_REASON, TRACE_EVENT } from './schema.js';
+
+/** In-page records kept before the recorder starts counting drops instead. */
+export const DEFAULT_MAX_QUEUED_MUTATIONS = 5000;
+
+/**
+ * Build the recorder's continuous stream.
+ *
+ * @param {Object} options - Collaborators the stream needs
+ * @param {Object} options.page - The page being traced
+ * @param {Object} options.bundle - Open trace bundle
+ * @param {Function} options.record - Writes one timeline record
+ * @param {Object} options.identity - Trace identity, see `identity.js`
+ * @param {Function} options.note - Reports something that could not be done
+ * @param {Object} options.domOptions - Resolved `dom` options
+ * @param {Object} options.privacyOptions - Resolved privacy options
+ * @param {Object} options.limits - Resolved limits
+ * @param {number} options.captureTimeoutMs - Budget for one drain
+ * @param {Function} options.evaluateInPage - Fallback for a page without frames
+ * @returns {Object} `{install, installPersistent, drain, stop}`
+ */
+export function createMutationStream(options) {
+  const {
+    page,
+    bundle,
+    record,
+    identity,
+    note,
+    domOptions,
+    privacyOptions,
+    limits,
+    captureTimeoutMs,
+    evaluateInPage,
+  } = options;
+
+  const enabled = Boolean(domOptions.mutations);
+
+  /** What the in-page recorder is built with, wherever it is installed. */
+  const recorderOptions = {
+    globalName: RECORDER_GLOBAL,
+    redactSelectors: privacyOptions.redactSelectors,
+    redacted: REDACTED,
+    maxQueued: limits.maxQueuedMutations ?? DEFAULT_MAX_QUEUED_MUTATIONS,
+    liveState: domOptions.liveState !== false,
+  };
+
+  /**
+   * Run something in every frame of the page.
+   *
+   * An iframe is a separate document with a separate observer, so a trace that
+   * only ever spoke to the main frame recorded nothing that happened inside
+   * one. A frame that goes away mid-call is expected rather than a failure: a
+   * frame that has already gone has nothing left to report.
+   *
+   * @param {Function} fn - The function to evaluate
+   * @param {*} argument - Its argument
+   * @returns {Promise<Array>} One result per frame that answered
+   */
+  async function evaluateInFrames(fn, argument) {
+    const frames = framesOf(page);
+    if (frames.length === 0) {
+      return [await evaluateInPage(fn, argument)];
+    }
+    const results = await Promise.all(
+      frames.map(async (frame) => {
+        try {
+          return await frame.evaluate(fn, argument);
+        } catch {
+          return null;
+        }
+      })
+    );
+    return results.filter((result) => result !== null);
+  }
+
+  async function dropped(
+    member,
+    error,
+    reason = TRACE_DROP_REASON.CAPTURE_FAILED
+  ) {
+    await bundle.drop({
+      reason,
+      member,
+      detail: error.message ?? String(error),
+    });
+  }
+
+  /**
+   * Install the recorder into the documents that already exist.
+   *
+   * New documents are covered by the init script; this covers the one the trace
+   * started in, and any frame that was already open when it did.
+   *
+   * @returns {Promise<void>} Resolves once every frame has been asked
+   */
+  async function install() {
+    if (!enabled) {
+      return;
+    }
+    try {
+      await evaluateInFrames(installMutationRecorderInPage, recorderOptions);
+    } catch (error) {
+      await dropped('mutation-recorder', error);
+    }
+  }
+
+  /**
+   * Register the recorder to run before the code of every future document.
+   *
+   * This is what closes the gap between a navigation and the next checkpoint:
+   * without it a new document's own scripts run unobserved, and everything they
+   * build looks, on replay, like it had been there all along.
+   *
+   * @returns {Promise<Function|null>} Detach, when the engine offers one
+   */
+  async function installPersistent() {
+    if (!enabled) {
+      return null;
+    }
+    try {
+      return await installInitScript({
+        page,
+        fn: installMutationRecorderInPage,
+        arg: recorderOptions,
+        note,
+      });
+    } catch (error) {
+      await dropped('mutation-recorder-init', error);
+      return null;
+    }
+  }
+
+  /**
+   * Take everything every frame has queued and write it as one interval.
+   *
+   * @param {number} index - Checkpoint the interval ends at
+   * @returns {Promise<string|null>} The member written, when there was anything
+   */
+  async function drain(index) {
+    if (!enabled) {
+      return null;
+    }
+    try {
+      const drained = await withDeadline(
+        evaluateInFrames(drainMutationsInPage, RECORDER_GLOBAL),
+        captureTimeoutMs,
+        'trace mutation drain'
+      );
+
+      const batches = [];
+      let over = 0;
+      for (const frame of drained) {
+        over += frame?.dropped ?? 0;
+        for (const batch of frame?.batches ?? []) {
+          // The identity of the trace comes from here and the identity of the
+          // document comes from inside it: a frame handle is not something an
+          // NDJSON record can hold, so each document names itself.
+          batches.push({ ...identity.owner(), ...batch });
+        }
+      }
+
+      if (over) {
+        await dropped(
+          'mutations',
+          new Error(`${over} records over the in-page queue limit`),
+          TRACE_DROP_REASON.SIZE_LIMIT
+        );
+      }
+
+      // Frames are drained in parallel, so the batches come back grouped by
+      // frame; one ordered file is what a replay reads.
+      batches.sort((left, right) => (left.at ?? 0) - (right.at ?? 0));
+
+      const member = await bundle.writeMutations(index, batches);
+      if (member) {
+        await record(TRACE_EVENT.MUTATIONS, {
+          member,
+          batches: batches.length,
+          checkpoint: index,
+          frames: drained.length,
+        });
+      }
+      return member;
+    } catch (error) {
+      await dropped('mutations', error);
+      return null;
+    }
+  }
+
+  /**
+   * Switch the recorder off in every document that has one.
+   *
+   * A recorder left observing would keep queueing into a queue nobody drains.
+   *
+   * @returns {Promise<void>} Resolves once every frame has been asked
+   */
+  async function stop() {
+    if (!enabled) {
+      return;
+    }
+    try {
+      await evaluateInFrames(stopMutationRecorderInPage, RECORDER_GLOBAL);
+    } catch (error) {
+      note?.(`could not stop the in-page recorder: ${error.message}`);
+    }
+  }
+
+  return { install, installPersistent, drain, stop };
+}

package/src/traces/observers.js

@@ -0,0 +1,272 @@
+/**
+ * The observers behind a trace's shared timeline (issue #87).
+ *
+ * Navigation, Browser Commander interactions, console output, page errors,
+ * dialogs, failed requests and downloads are reported by different objects in
+ * different engines. They all end up as records on one ordered timeline, and
+ * this module is the only place that knows how each of them is reached.
+ */
+
+import { TRACE_EVENT } from './schema.js';
+
+/** Commander methods recorded as interactions while a trace is running. */
+export const TRACED_INTERACTIONS = Object.freeze([
+  'click',
+  'clickButton',
+  'fill',
+  'fillTextArea',
+  'goto',
+  'pressKey',
+  'typeText',
+  'setContent',
+  'scrollIntoView',
+]);
+
+/**
+ * What an interaction was aimed at.
+ *
+ * A selector and a URL are the useful part of an interaction; typed text is
+ * never taken, because that is where passwords are. Most commander methods
+ * take an options object, and a few legacy ones take the selector directly,
+ * so both shapes are read here.
+ *
+ * @param {*} argument - The interaction's first argument
+ * @returns {string|null} The selector or URL it acted on
+ */
+export function interactionTarget(argument) {
+  if (typeof argument === 'string') {
+    return argument;
+  }
+  if (!argument || typeof argument !== 'object') {
+    return null;
+  }
+  for (const key of ['selector', 'url', 'locatorOrElement']) {
+    if (typeof argument[key] === 'string') {
+      return argument[key];
+    }
+  }
+  return null;
+}
+
+/**
+ * Subscribe to everything a running trace listens to.
+ *
+ * @param {Object} options - `{commander, page, eventSources, record, note, identity}`
+ * @param {Object} [options.commander] - The commander being traced, when there
+ *   is one; interactions and managed downloads are reached through it
+ * @param {Object} options.page - The page being traced
+ * @param {string[]} options.eventSources - Sources the caller asked for
+ * @param {Function} options.record - Writes one timeline record
+ * @param {Function} options.note - Reports an observer that could not attach
+ * @param {Object} [options.identity] - Trace identity, see `identity.js`; the
+ *   navigation observer is what moves it on to the next navigation, and the
+ *   interaction observer is what names an action (issue #93)
+ * @returns {Function[]} One detach function per attached observer
+ */
+export function attachTimelineObservers(options) {
+  const { commander, page, eventSources, record, note, identity } = options;
+  const detachers = [];
+
+  function subscribe(source, attach) {
+    if (!eventSources.includes(source)) {
+      return;
+    }
+    try {
+      const detach = attach();
+      if (typeof detach === 'function') {
+        detachers.push(detach);
+      }
+    } catch (error) {
+      note(`could not observe ${source}: ${error.message}`);
+    }
+  }
+
+  function onPage(eventName, listener) {
+    page.on?.(eventName, listener);
+    return () =>
+      page.off?.(eventName, listener) ??
+      page.removeListener?.(eventName, listener);
+  }
+
+  subscribe('navigation', () => {
+    const manager = commander?.navigationManager;
+    if (!manager) {
+      // Without the manager the engine's own event still gives ordering.
+      return onPage('framenavigated', (frame) => {
+        const isMain =
+          typeof frame?.parentFrame === 'function'
+            ? frame.parentFrame() === null
+            : true;
+        // A record's navigation is the one it happened during, so the counter
+        // moves on before the record that reports the move is written.
+        const navigationId = isMain
+          ? identity?.navigated()
+          : identity?.navigationId;
+        void record(TRACE_EVENT.NAVIGATION, {
+          phase: 'framenavigated',
+          navigationId,
+          mainFrame: isMain,
+          url: typeof frame?.url === 'function' ? frame.url() : String(frame),
+        });
+      });
+    }
+    const started = (info) =>
+      void record(TRACE_EVENT.NAVIGATION, { phase: 'start', ...info });
+    const completed = (info) =>
+      void record(TRACE_EVENT.NAVIGATION, {
+        phase: 'complete',
+        // Everything recorded after this belongs to the document that just
+        // loaded, which is the distinction a replay needs to stop mixing two
+        // pages' changes together (issue #93).
+        navigationId: identity?.navigated(),
+        ...info,
+      });
+    const changed = (info) =>
+      void record(TRACE_EVENT.NAVIGATION, { phase: 'urlchange', ...info });
+    manager.on('onNavigationStart', started);
+    manager.on('onNavigationComplete', completed);
+    manager.on('onUrlChange', changed);
+    return () => {
+      manager.off?.('onNavigationStart', started);
+      manager.off?.('onNavigationComplete', completed);
+      manager.off?.('onUrlChange', changed);
+    };
+  });
+
+  subscribe('console', () =>
+    onPage('console', (message) => {
+      void record(TRACE_EVENT.CONSOLE, {
+        level: typeof message?.type === 'function' ? message.type() : null,
+        text:
+          typeof message?.text === 'function'
+            ? message.text()
+            : String(message),
+      });
+    })
+  );
+
+  subscribe('pageerror', () =>
+    onPage('pageerror', (error) => {
+      void record(TRACE_EVENT.PAGE_ERROR, {
+        message: error?.message ?? String(error),
+        stack: error?.stack ?? null,
+      });
+    })
+  );
+
+  subscribe('requestfailed', () =>
+    onPage('requestfailed', (request) => {
+      void record(TRACE_EVENT.REQUEST_FAILED, {
+        url: typeof request?.url === 'function' ? request.url() : null,
+        method: typeof request?.method === 'function' ? request.method() : null,
+        failure:
+          typeof request?.failure === 'function'
+            ? (request.failure()?.errorText ?? null)
+            : null,
+      });
+    })
+  );
+
+  subscribe('dialog', () => {
+    const manager = commander?.dialogManager;
+    if (!manager?.observeDialogs && !manager?.onDialog) {
+      return null;
+    }
+    const listener = (dialog) => {
+      void record(TRACE_EVENT.DIALOG, {
+        type: typeof dialog?.type === 'function' ? dialog.type() : null,
+        message:
+          typeof dialog?.message === 'function' ? dialog.message() : null,
+      });
+    };
+    // Recording a dialog must not decide what happens to it: a passive
+    // observer leaves the manager's auto-dismissal in place, so a traced run
+    // answers dialogs exactly the way the same untraced run would.
+    if (manager.observeDialogs) {
+      manager.observeDialogs(listener);
+      return () => manager.unobserveDialogs?.(listener);
+    }
+    manager.onDialog(listener);
+    return () => manager.offDialog?.(listener);
+  });
+
+  subscribe('download', () => {
+    const downloads = commander?.downloads;
+    if (!downloads?.on) {
+      return null;
+    }
+    const listeners = ['started', 'completed', 'failed', 'cancelled'].map(
+      (phase) => {
+        const listener = (artifact) => {
+          void record(TRACE_EVENT.DOWNLOAD, {
+            phase,
+            id: artifact?.id ?? null,
+            // The path and checksum are the reference; the bytes stay where
+            // the download manager put them.
+            suggestedFilename: artifact?.suggestedFilename ?? null,
+            path: artifact?.path ?? null,
+            checksum: artifact?.checksum ?? null,
+            bytes: artifact?.bytes ?? null,
+            url: artifact?.url ?? null,
+            failure: artifact?.failure ?? null,
+          });
+        };
+        downloads.on(phase, listener);
+        return [phase, listener];
+      }
+    );
+    return () => {
+      for (const [phase, listener] of listeners) {
+        downloads.off?.(phase, listener);
+      }
+    };
+  });
+
+  subscribe('interaction', () => {
+    if (!commander) {
+      return null;
+    }
+    const originals = new Map();
+    for (const name of TRACED_INTERACTIONS) {
+      const original = commander[name];
+      if (typeof original !== 'function') {
+        continue;
+      }
+      originals.set(name, original);
+      commander[name] = async (...args) => {
+        const startedMs = Date.now();
+        const target = interactionTarget(args[0]);
+        // Named before the call, so everything the action causes can be traced
+        // back to it even when the action itself ends in an exception.
+        const actionId = identity?.nextActionId() ?? null;
+        try {
+          const result = await original(...args);
+          await record(TRACE_EVENT.INTERACTION, {
+            actionId,
+            action: name,
+            target,
+            durationMs: Date.now() - startedMs,
+            ok: true,
+          });
+          return result;
+        } catch (error) {
+          await record(TRACE_EVENT.INTERACTION, {
+            actionId,
+            action: name,
+            target,
+            durationMs: Date.now() - startedMs,
+            ok: false,
+            error: error.message,
+          });
+          throw error;
+        }
+      };
+    }
+    return () => {
+      for (const [name, original] of originals) {
+        commander[name] = original;
+      }
+    };
+  });
+  return detachers;
+}