browser-commander 0.28.0 → 0.30.0

package/src/traces/page-capture.js

@@ -28,6 +28,7 @@ export function captureSnapshotInPage(options) {
   const {
     redactSelectors = [],
     redactAttributes = [],
+    useDefaults = true,
     redacted = '[redacted]',
     html: wantHtml = true,
     liveControlState = true,
@@ -121,6 +122,13 @@ export function captureSnapshotInPage(options) {
   // `<input>` otherwise shows the value the server sent, not the one the user
   // typed.
   const carryLiveState = (original, copy, secret) => {
+    if (
+      useDefaults &&
+      original.localName === 'meta' &&
+      /token|csrf|xsrf|secret/i.test(original.getAttribute('name') || '')
+    ) {
+      copy.setAttribute('content', redacted);
+    }
     for (const attribute of redactAttributes) {
       if (copy.hasAttribute && copy.hasAttribute(attribute)) {
         copy.setAttribute(attribute, redacted);
@@ -197,8 +205,13 @@ export function captureSnapshotInPage(options) {
     const clone = cloneWithState(document.documentElement);
 
     html = `<!DOCTYPE html>\n${clone?.outerHTML ?? ''}`;
-    if (maxHtmlBytes > 0 && html.length > maxHtmlBytes) {
-      html = html.slice(0, maxHtmlBytes);
+    const encoded = new globalThis.TextEncoder().encode(html);
+    if (maxHtmlBytes > 0 && encoded.length > maxHtmlBytes) {
+      const marker = '<!-- trace HTML truncated -->';
+      const prefix = new globalThis.TextDecoder().decode(
+        encoded.slice(0, Math.max(0, maxHtmlBytes - marker.length - 3))
+      );
+      html = `${prefix}${marker}`;
       truncated = true;
     }
   }
@@ -281,6 +294,8 @@ export function installMutationRecorderInPage(options) {
   const {
     globalName = '__browserCommanderTrace__',
     redactSelectors = [],
+    redactAttributes = [],
+    useDefaults = true,
     redacted = '[redacted]',
     maxQueued = 5000,
     ignoreSelectors = [],
@@ -296,8 +311,6 @@ export function installMutationRecorderInPage(options) {
     queue: [],
     dropped: 0,
     sequence: 0,
-    // Every document gets its own name, so a batch drained from an iframe is
-    // never mistaken for one drained from the page that holds it.
     frameId: `frame-${Math.random().toString(36).slice(2, 10)}`,
     frameName: window.name || null,
     mainFrame: window.top === window,
@@ -317,7 +330,11 @@ export function installMutationRecorderInPage(options) {
       }
     });
   };
-  const isSecret = (node) => matchesAny(node, redactSelectors);
+  const isSecret = (node) =>
+    matchesAny(node, redactSelectors) ||
+    (useDefaults &&
+      node?.localName === 'meta' &&
+      /token|csrf|xsrf|secret|password/i.test(node.getAttribute('name') ?? ''));
   const isIgnored = (node) => matchesAny(node, ignoreSelectors);
   const sanitized = (node) => {
     const clone = node.cloneNode(true);
@@ -334,6 +351,20 @@ export function installMutationRecorderInPage(options) {
         }
       }
     }
+    for (const element of [clone, ...clone.querySelectorAll('*')]) {
+      for (const attribute of redactAttributes) {
+        if (element.hasAttribute(attribute)) {
+          element.setAttribute(attribute, redacted);
+        }
+      }
+      if (
+        useDefaults &&
+        element.localName === 'meta' &&
+        /token|csrf|xsrf|secret/i.test(element.getAttribute('name') || '')
+      ) {
+        element.setAttribute('content', redacted);
+      }
+    }
     return clone;
   };
 
@@ -346,11 +377,9 @@ export function installMutationRecorderInPage(options) {
         steps.unshift(`#${at.id}`);
         break;
       }
-      const siblings = at.parentElement
-        ? [...at.parentElement.children].filter(
-            (c) => c.localName === at.localName
-          )
-        : [];
+      const siblings = [...(at.parentElement?.children ?? [])].filter(
+        (c) => c.localName === at.localName
+      );
       const nth =
         siblings.length > 1 ? `:nth-of-type(${siblings.indexOf(at) + 1})` : '';
       steps.unshift(at.localName + nth);
@@ -399,6 +428,23 @@ export function installMutationRecorderInPage(options) {
     };
   };
 
+  // Mutation targets identify an existing node. Only additions need markup.
+  const identify = (node) => {
+    if (!node || isIgnored(node)) {
+      return null;
+    }
+    const element = node.nodeType === 1 ? node : node.parentElement;
+    return {
+      type: node.nodeType === 3 ? 'text' : 'element',
+      path: pathOf(element),
+      tag: element?.localName,
+      id: element?.id || null,
+      visible:
+        Boolean(element?.getClientRects().length) &&
+        window.getComputedStyle(element).visibility !== 'hidden',
+    };
+  };
+
   /**
    * Describe a node that is no longer where it was.
    *
@@ -458,13 +504,15 @@ export function installMutationRecorderInPage(options) {
         .map((record) => {
           const entry = {
             kind: record.type,
-            target: describe(record.target),
+            target: identify(record.target),
           };
           if (record.type === 'attributes') {
             entry.attribute = record.attributeName;
             const secret =
               isSecret(record.target) ||
-              redactSelectors.includes(record.attributeName);
+              redactAttributes.includes(record.attributeName) ||
+              (useDefaults &&
+                /token|csrf|xsrf|secret|password/i.test(record.attributeName));
             entry.before = secret ? redacted : record.oldValue;
             entry.after =
               secret || !record.target.getAttribute
@@ -485,8 +533,8 @@ export function installMutationRecorderInPage(options) {
             entry.removed = [...record.removedNodes]
               .filter((node) => !isIgnored(node))
               .map((node) => describeAt(node, record));
-            entry.previous = describe(record.previousSibling);
-            entry.next = describe(record.nextSibling);
+            entry.previous = identify(record.previousSibling);
+            entry.next = identify(record.nextSibling);
           }
           return entry;
         }),
@@ -511,7 +559,6 @@ export function installMutationRecorderInPage(options) {
     // into a field and then leaving it fires `input` for each keystroke and
     // `change` for the value it ended at, and a checkbox fires both for the
     // one click. Replaying a state that is already on screen shows nothing, so
-    // a repeat of what this element last reported is not recorded.
     const repeats = (element, property, after) => {
       const seen = lastLive.get(element) || {};
       const asText = JSON.stringify(after === undefined ? null : after);
@@ -526,16 +573,13 @@ export function installMutationRecorderInPage(options) {
     const liveEntry = (target, property, after, before) => ({
       kind: 'live-state',
       property,
-      target: describe(target),
+      target: identify(target),
       before: before === undefined ? null : before,
       after,
     });
 
     const pushLive = (entry) => {
-      // Scrolling fires far more often than it changes anything worth
-      // replaying, so consecutive scrolls of one element collapse into the
-      // position it ended at. Every other property keeps each step, because
-      // "stepwise replay must show each state" (issue #93).
+      // Consecutive scrolls collapse; other state changes retain each step.
       const last = state.queue[state.queue.length - 1];
       const only = last && last.records.length === 1 ? last.records[0] : null;
       if (
@@ -608,7 +652,7 @@ export function installMutationRecorderInPage(options) {
       pushLive({
         kind: 'live-state',
         property: 'scroll',
-        target: scrolling ? describe(scrolling) : null,
+        target: scrolling ? identify(scrolling) : null,
         before: null,
         after: scrolling
           ? { top: scrolling.scrollTop, left: scrolling.scrollLeft }

package/src/traces/recorder.js

@@ -1,5 +1,6 @@
 import { abortTraceStart } from './start-cleanup.js';
 import { createTraceStop } from './stop.js';
+import { openRollingBundle } from './rolling-bundle.js';
 /**
  * The trace recorder (issue #87).
  *
@@ -15,10 +16,13 @@ import { openTraceLinks } from './links.js';
 import { createTraceIdentity } from './identity.js';
 import { withDeadline } from './deadline.js';
 import { createMutationStream } from './mutation-stream.js';
-import { attachTimelineObservers } from './observers.js';
+import { createUserActionStream } from './user-actions.js';
+import { startTraceDrain } from './drain-timer.js';
+import { attachRecorderObservers } from './observers.js';
 import { attachNetwork, validateNetworkOptions } from './network.js';
 import { startRecording } from '../capture/recording.js';
 import { normalizePrivacyOptions, redactValue } from './redaction.js';
+import path from 'node:path';
 import {
   DEFAULT_CAPTURE_TIMEOUT,
   TRACE_CHECKPOINT_REASON,
@@ -93,13 +97,19 @@ function normalizeTraceOptions(options) {
   return {
     ...options,
     page: options.page ?? options.commander?.page,
-    mode: options.mode ?? TRACE_MODE.CHECKPOINTS,
+    mode: options.mode ?? TRACE_MODE.CONTINUOUS,
     screenshots: options.screenshots ?? 'checkpoints',
     dom: options.dom ?? {},
     privacy: options.privacy ?? {},
     limits: options.limits ?? {},
-    links: options.links ?? null,
-    network: options.network ?? false,
+    links:
+      options.links === false || options.links === null
+        ? null
+        : (options.links ?? {
+            output: path.join(options.output ?? '', 'trace.lino'),
+            dom: 'full',
+          }),
+    network: options.network ?? { har: true },
     strict: options.strict ?? false,
     captureTimeoutMs: options.captureTimeoutMs ?? DEFAULT_CAPTURE_TIMEOUT,
     commanderVersion: options.commanderVersion ?? null,
@@ -109,6 +119,7 @@ function normalizeTraceOptions(options) {
 }
 
 export async function startTrace(options = {}) {
+  options = normalizeTraceOptions(options);
   if (options.limits?.rotate) {
     const { startRollingTrace } = await import('./rolling.js');
     return startRollingTrace(options, startTrace);
@@ -150,8 +161,8 @@ export async function startTrace(options = {}) {
   // Otherwise a rejected start (for example `links: {}`) strands the bundle's
   // events handle because no running trace is returned for the caller to stop.
   validateLinksOptions(links);
-  const engine = commander?.engine ?? null;
-  const identity = createTraceIdentity({ page });
+  const engine = commander?.engine ?? null,
+    identity = createTraceIdentity({ page });
 
   // A continuous trace records changes, and a change is only meaningful against
   // something. Taking the base snapshot automatically is the safe default,
@@ -166,13 +177,13 @@ export async function startTrace(options = {}) {
   // readable export of everything that had happened. The bundle stays
   // authoritative: nothing is written here that is not written there first.
   let linksSink = null;
-  const bundle = await openTraceBundle({
+  const bundle = await openRecorderBundle(options, {
     output,
     limits,
     strict,
     now,
     monotonic,
-    onEvent: (event) => linksSink?.event(event),
+    onEvent: (event, source) => linksSink?.event(event, source),
   });
   const startedAt = new Date(now()).toISOString();
 
@@ -242,6 +253,18 @@ export async function startTrace(options = {}) {
     captureTimeoutMs,
     evaluateInPage,
   });
+  const userActions = createUserActionStream({
+    page,
+    record,
+    identity,
+    note,
+    privacyOptions,
+    limits,
+    enabled: eventSources.includes('interaction'),
+    ignoreSelectors: domOptions.ignoreSelectors,
+    evaluateInPage,
+    captureTimeoutMs,
+  });
 
   async function screenshot(reason) {
     const wanted =
@@ -288,7 +311,7 @@ export async function startTrace(options = {}) {
     detachNetwork = async () => {},
     video = null;
   try {
-    detachers = attachTimelineObservers({
+    const observers = attachRecorderObservers({
       commander,
       page,
       eventSources,
@@ -297,7 +320,14 @@ export async function startTrace(options = {}) {
       identity,
       now,
     });
-    detachNetwork = attachNetwork({ page, network, record, note });
+    detachers = observers.detachers;
+    detachNetwork = attachNetwork({
+      page,
+      network,
+      record,
+      note,
+      privacy: privacyOptions,
+    });
     video = options.video
       ? await startRecording({
           page,
@@ -337,7 +367,11 @@ export async function startTrace(options = {}) {
       events: eventSources,
       initialCheckpoint: Boolean(baseCheckpoint),
     });
-    await mutations.install();
+    await Promise.all([mutations.install(), userActions.install()]);
+    detachers.push(async () => {
+      await userActions.drain();
+      await userActions.stop();
+    });
 
     if (baseCheckpoint) {
       await checkpoint(
@@ -349,23 +383,10 @@ export async function startTrace(options = {}) {
       );
     }
 
-    let drainTask;
-    if (domOptions.mutations) {
-      const timer = setInterval(() => {
-        if (!drainTask) {
-          drainTask = mutations
-            .drain(captureState.index)
-            .catch((error) => note(error.message))
-            .finally(() => {
-              drainTask = null;
-            });
-        }
-      }, 500);
-      timer.unref?.();
-      detachers.push(async () => {
-        clearInterval(timer);
-        await drainTask;
-      });
+    if (domOptions.mutations || eventSources.includes('interaction')) {
+      detachers.push(
+        startTraceDrain({ mutations, userActions, captureState, note })
+      );
     }
 
     /**
@@ -405,12 +426,19 @@ export async function startTrace(options = {}) {
 
     return {
       path: bundle.root,
+      get currentPath() {
+        return bundle.currentPath ?? bundle.root;
+      },
+      get segments() {
+        return bundle.segments;
+      },
       links: linksSink?.path ?? null,
       mode: recorderMode,
       get bytesWritten() {
         return bundle.bytesWritten;
       },
       checkpoint,
+      attachCommander: observers.attachCommander,
       /**
        * Record an event a caller cares about on the same timeline.
        *
@@ -447,5 +475,28 @@ export async function startTrace(options = {}) {
 
 function requireTraceSupport(options) {
   validateNetworkOptions(options.network);
+  const maxQueued = options.limits.maxQueuedUserActions ?? 5000;
+  if (!Number.isSafeInteger(maxQueued) || maxQueued < 1) {
+    throw new RangeError(
+      'maxQueuedUserActions must be a positive safe integer'
+    );
+  }
   (options.page ?? options.commander?.page)?.requireFeature?.('tracing');
 }
+
+function openRecorderBundle(options, settings) {
+  const writer =
+    options.openBundle ??
+    (options.mode === TRACE_MODE.CONTINUOUS
+      ? (config) =>
+          openRollingBundle(
+            config,
+            {
+              lazy: true,
+              maxBytes: settings.limits.maxBundleBytes ?? 256 * 1024 * 1024,
+            },
+            options
+          )
+      : openTraceBundle);
+  return writer(settings);
+}

package/src/traces/redaction.js

@@ -13,6 +13,7 @@ export const REDACTED = '[redacted]';
 /** Controls whose value is a secret unless the caller says otherwise. */
 export const DEFAULT_REDACT_SELECTORS = Object.freeze([
   'input[type=password]',
+  'input[type=hidden]',
   '[data-private]',
   '[data-bc-redact]',
 ]);
@@ -42,8 +43,74 @@ export const DEFAULT_REDACT_QUERY_PARAMS = Object.freeze([
   'session',
   'signature',
   'token',
+  'csrf',
+  'xsrf',
+  '_xsrf',
 ]);
 
+/** Match names rather than values, including framework-specific token names. */
+export function sensitiveName(name, privacy) {
+  return (
+    Boolean(privacy.useDefaults) &&
+    /password|passwd|token|csrf|xsrf|otp|secret|api[_-]?key|authorization/i.test(
+      name
+    )
+  );
+}
+
+function redactFields(value, privacy) {
+  if (Array.isArray(value)) {
+    return value.map((entry) => redactFields(entry, privacy));
+  }
+  if (value && typeof value === 'object') {
+    return Object.fromEntries(
+      Object.entries(value).map(([key, entry]) => [
+        key,
+        sensitiveName(key, privacy) ? REDACTED : redactFields(entry, privacy),
+      ])
+    );
+  }
+  return value;
+}
+
+function redactStructuredText(value, privacy) {
+  if (!privacy.useDefaults) {
+    return value;
+  }
+  try {
+    const parsed = JSON.parse(value);
+    if (parsed && typeof parsed === 'object') {
+      return JSON.stringify(redactFields(parsed, privacy));
+    }
+  } catch {
+    /* HTML, forms and script assignments are handled below. */
+  }
+  return value
+    .replace(
+      /(\b(?:["']?[\w-]+["']?)\s*[:=]\s*)("(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\d+(?:\.\d+)?)/g,
+      (match, prefix) =>
+        sensitiveName(prefix, privacy)
+          ? `${prefix}${JSON.stringify(REDACTED)}`
+          : match
+    )
+    .replace(/(^|[&?])([^=&\s]+)=([^&\s]*)/g, (match, start, key) => {
+      let name = key;
+      try {
+        name = decodeURIComponent(key.replaceAll('+', ' '));
+      } catch {
+        /* Keep malformed names. */
+      }
+      return sensitiveName(name, privacy)
+        ? `${start}${key}=${REDACTED}`
+        : match;
+    })
+    .replace(
+      /(Content-Disposition:[^\r\n]*?;\s*name="([^"]+)"[^\r\n]*\r?\n(?:[^\r\n]+\r?\n)*\r?\n)([\s\S]*?)(?=\r?\n--)/gi,
+      (match, header, name) =>
+        sensitiveName(name, privacy) ? `${header}${REDACTED}` : match
+    );
+}
+
 function lowerSet(values) {
   return [...new Set(values.map((value) => String(value).toLowerCase()))];
 }
@@ -75,6 +142,7 @@ export function normalizePrivacyOptions(privacy = {}) {
   }
 
   return {
+    useDefaults,
     redactSelectors: lowerSet([
       ...(useDefaults ? DEFAULT_REDACT_SELECTORS : []),
       ...redactSelectors,
@@ -107,7 +175,8 @@ export function redactText(value, privacy, context = {}) {
     return value;
   }
 
-  let text = value;
+  let text =
+    context.kind === 'url' ? value : redactStructuredText(value, privacy);
   for (const pattern of privacy.redactPatterns || []) {
     // A caller's regex may be stateful; resetting keeps one trace's output
     // from depending on the previous record's match position.
@@ -148,7 +217,10 @@ export function redactUrl(url, privacy) {
       parsed.password = parsed.password ? REDACTED : '';
     }
     for (const key of [...parsed.searchParams.keys()]) {
-      if (privacy.redactQueryParams.includes(key.toLowerCase())) {
+      if (
+        privacy.redactQueryParams.includes(key.toLowerCase()) ||
+        sensitiveName(key, privacy)
+      ) {
         parsed.searchParams.set(key, REDACTED);
       }
     }
@@ -157,7 +229,10 @@ export function redactUrl(url, privacy) {
       const fragment = new URLSearchParams(parsed.hash.slice(1));
       let changed = false;
       for (const key of [...fragment.keys()]) {
-        if (privacy.redactQueryParams.includes(key.toLowerCase())) {
+        if (
+          privacy.redactQueryParams.includes(key.toLowerCase()) ||
+          sensitiveName(key, privacy)
+        ) {
           fragment.set(key, REDACTED);
           changed = true;
         }
@@ -190,9 +265,11 @@ export function redactHeaders(headers, privacy) {
 
   const result = {};
   for (const [name, value] of Object.entries(headers)) {
-    result[name] = privacy.redactAttributes.includes(name.toLowerCase())
-      ? REDACTED
-      : redactText(String(value), privacy, { kind: 'header', name });
+    result[name] =
+      privacy.redactAttributes.includes(name.toLowerCase()) ||
+      sensitiveName(name, privacy)
+        ? REDACTED
+        : redactText(String(value), privacy, { kind: 'header', name });
   }
   return result;
 }
@@ -209,6 +286,12 @@ export function redactHeaders(headers, privacy) {
  * @returns {*} The value with strings redacted
  */
 export function redactValue(value, privacy, key = '') {
+  if (
+    sensitiveName(key, privacy) ||
+    privacy.redactAttributes.includes(key.toLowerCase())
+  ) {
+    return REDACTED;
+  }
   if (typeof value === 'string') {
     if (privacy.redactAttributes.includes(key.toLowerCase())) {
       return REDACTED;