browser-commander 0.28.0 → 0.29.0

package/src/traces/mutation-stream.js

@@ -20,7 +20,7 @@ import {
   RECORDER_GLOBAL,
   stopMutationRecorderInPage,
 } from './page-capture.js';
-import { REDACTED } from './redaction.js';
+import { redactValue, REDACTED } from './redaction.js';
 import { TRACE_DROP_REASON, TRACE_EVENT } from './schema.js';
 
 /** In-page records kept before the recorder starts counting drops instead. */
@@ -62,6 +62,8 @@ export function createMutationStream(options) {
   const recorderOptions = {
     globalName: RECORDER_GLOBAL,
     redactSelectors: privacyOptions.redactSelectors,
+    redactAttributes: privacyOptions.redactAttributes,
+    useDefaults: privacyOptions.useDefaults,
     redacted: REDACTED,
     ignoreSelectors: domOptions.ignoreSelectors ?? [],
     maxQueued: limits.maxQueuedMutations ?? DEFAULT_MAX_QUEUED_MUTATIONS,
@@ -195,13 +197,17 @@ export function createMutationStream(options) {
       // frame; one ordered file is what a replay reads.
       batches.sort((left, right) => (left.at ?? 0) - (right.at ?? 0));
 
-      const member = await bundle.writeMutations(index, batches);
+      const member = await bundle.writeMutations(
+        index,
+        redactValue(batches, privacyOptions)
+      );
       if (member) {
         await record(TRACE_EVENT.MUTATIONS, {
           member,
           batches: batches.length,
           checkpoint: index,
           frames: drained.length,
+          truncated: bundle.mutationTruncated,
         });
       }
       return member;

package/src/traces/network.js

@@ -1,6 +1,11 @@
 import fs from 'node:fs/promises';
 import path from 'node:path';
 import { withDeadline } from './deadline.js';
+import {
+  normalizePrivacyOptions,
+  redactText,
+  redactHeaders as privateHeaders,
+} from './redaction.js';
 
 const PRIVATE_HEADERS = new Set([
   'cookie',
@@ -9,11 +14,20 @@ const PRIVATE_HEADERS = new Set([
   'set-cookie',
 ]);
 export function redactHeaders(headers = {}) {
-  return Object.fromEntries(
-    Object.entries(headers).map(([key, value]) => [
-      key.toLowerCase(),
-      PRIVATE_HEADERS.has(key.toLowerCase()) ? '[redacted]' : value,
-    ])
+  return privateHeaders(
+    Object.fromEntries(
+      Object.entries(headers).map(([key, value]) => [
+        key.toLowerCase(),
+        PRIVATE_HEADERS.has(key.toLowerCase()) ? '[redacted]' : value,
+      ])
+    ),
+    normalizePrivacyOptions()
+  );
+}
+
+export function isTextBody(contentType = '') {
+  return /^(?:text\/|application\/(?:[\w.+-]*json|[\w.+-]*xml|javascript|x-www-form-urlencoded))|multipart\/form-data/i.test(
+    contentType
   );
 }
 
@@ -81,10 +95,12 @@ export function networkHar(events) {
           content: {
             size: response.body?.size ?? 0,
             mimeType: response.contentType ?? '',
-            ...(response.body
+            ...(response.body?.data !== undefined
               ? {
                   text: response.body.data,
-                  encoding: 'base64',
+                  ...(response.body.encoding === 'utf8'
+                    ? {}
+                    : { encoding: 'base64' }),
                   _truncated: response.body.truncated,
                 }
               : {}),
@@ -131,7 +147,13 @@ export function validateNetworkOptions(network) {
     );
   }
 }
-export function attachNetwork({ page, network, record, note }) {
+export function attachNetwork({
+  page,
+  network,
+  record,
+  note,
+  privacy = normalizePrivacyOptions(),
+}) {
   validateNetworkOptions(network);
   if (!network) {
     return async () => {};
@@ -172,7 +194,12 @@ export function attachNetwork({ page, network, record, note }) {
         ['document', 'xhr', 'fetch'].includes(request.resourceType?.()) &&
         request.postData?.()
           ? {
-              postData: Buffer.from(request.postData())
+              postData: Buffer.from(
+                redactText(request.postData(), privacy, {
+                  kind: 'network',
+                  name: 'postData',
+                })
+              )
                 .subarray(0, maxBodyBytes)
                 .toString('utf8'),
               postDataTruncated:
@@ -210,9 +237,26 @@ export function attachNetwork({ page, network, record, note }) {
           options.bodies &&
           ['document', 'xhr', 'fetch'].includes(resourceType)
         ) {
-          const length = Number(headers['content-length']);
-          if (Number.isFinite(length) && length > maxBodyBytes) {
-            payload.body = { size: length, truncated: true, data: '' };
+          let length = Number(headers['content-length']);
+          if (!Number.isFinite(length) && request.sizes) {
+            const sizes = await withDeadline(
+              request.sizes(),
+              5000,
+              'network sizes'
+            ).catch(() => ({}));
+            length = sizes.responseBodySize ?? NaN;
+          }
+          if (!isTextBody(payload.contentType ?? '')) {
+            payload.body = {
+              size: Number.isFinite(length) ? length : null,
+              omitted: 'binary',
+            };
+          } else if (Number.isFinite(length) && length > maxBodyBytes) {
+            payload.body = {
+              size: length,
+              truncated: true,
+              omitted: 'size limit',
+            };
           } else {
             try {
               const data = Buffer.from(
@@ -225,7 +269,15 @@ export function attachNetwork({ page, network, record, note }) {
               payload.body = {
                 size: data.length,
                 truncated: data.length > maxBodyBytes,
-                data: data.subarray(0, maxBodyBytes).toString('base64'),
+                encoding: 'utf8',
+                data: Buffer.from(
+                  redactText(data.toString('utf8'), privacy, {
+                    kind: 'network',
+                    name: 'responseBody',
+                  })
+                )
+                  .subarray(0, maxBodyBytes)
+                  .toString('utf8'),
               };
             } catch (error) {
               payload.bodyError = error.message;

package/src/traces/page-capture.js

@@ -28,6 +28,7 @@ export function captureSnapshotInPage(options) {
   const {
     redactSelectors = [],
     redactAttributes = [],
+    useDefaults = true,
     redacted = '[redacted]',
     html: wantHtml = true,
     liveControlState = true,
@@ -121,6 +122,13 @@ export function captureSnapshotInPage(options) {
   // `<input>` otherwise shows the value the server sent, not the one the user
   // typed.
   const carryLiveState = (original, copy, secret) => {
+    if (
+      useDefaults &&
+      original.localName === 'meta' &&
+      /token|csrf|xsrf|secret/i.test(original.getAttribute('name') || '')
+    ) {
+      copy.setAttribute('content', redacted);
+    }
     for (const attribute of redactAttributes) {
       if (copy.hasAttribute && copy.hasAttribute(attribute)) {
         copy.setAttribute(attribute, redacted);
@@ -197,8 +205,13 @@ export function captureSnapshotInPage(options) {
     const clone = cloneWithState(document.documentElement);
 
     html = `<!DOCTYPE html>\n${clone?.outerHTML ?? ''}`;
-    if (maxHtmlBytes > 0 && html.length > maxHtmlBytes) {
-      html = html.slice(0, maxHtmlBytes);
+    const encoded = new globalThis.TextEncoder().encode(html);
+    if (maxHtmlBytes > 0 && encoded.length > maxHtmlBytes) {
+      const marker = '<!-- trace HTML truncated -->';
+      const prefix = new globalThis.TextDecoder().decode(
+        encoded.slice(0, Math.max(0, maxHtmlBytes - marker.length - 3))
+      );
+      html = `${prefix}${marker}`;
       truncated = true;
     }
   }
@@ -281,6 +294,8 @@ export function installMutationRecorderInPage(options) {
   const {
     globalName = '__browserCommanderTrace__',
     redactSelectors = [],
+    redactAttributes = [],
+    useDefaults = true,
     redacted = '[redacted]',
     maxQueued = 5000,
     ignoreSelectors = [],
@@ -296,8 +311,6 @@ export function installMutationRecorderInPage(options) {
     queue: [],
     dropped: 0,
     sequence: 0,
-    // Every document gets its own name, so a batch drained from an iframe is
-    // never mistaken for one drained from the page that holds it.
     frameId: `frame-${Math.random().toString(36).slice(2, 10)}`,
     frameName: window.name || null,
     mainFrame: window.top === window,
@@ -317,7 +330,11 @@ export function installMutationRecorderInPage(options) {
       }
     });
   };
-  const isSecret = (node) => matchesAny(node, redactSelectors);
+  const isSecret = (node) =>
+    matchesAny(node, redactSelectors) ||
+    (useDefaults &&
+      node?.localName === 'meta' &&
+      /token|csrf|xsrf|secret|password/i.test(node.getAttribute('name') ?? ''));
   const isIgnored = (node) => matchesAny(node, ignoreSelectors);
   const sanitized = (node) => {
     const clone = node.cloneNode(true);
@@ -334,6 +351,20 @@ export function installMutationRecorderInPage(options) {
         }
       }
     }
+    for (const element of [clone, ...clone.querySelectorAll('*')]) {
+      for (const attribute of redactAttributes) {
+        if (element.hasAttribute(attribute)) {
+          element.setAttribute(attribute, redacted);
+        }
+      }
+      if (
+        useDefaults &&
+        element.localName === 'meta' &&
+        /token|csrf|xsrf|secret/i.test(element.getAttribute('name') || '')
+      ) {
+        element.setAttribute('content', redacted);
+      }
+    }
     return clone;
   };
 
@@ -346,11 +377,9 @@ export function installMutationRecorderInPage(options) {
         steps.unshift(`#${at.id}`);
         break;
       }
-      const siblings = at.parentElement
-        ? [...at.parentElement.children].filter(
-            (c) => c.localName === at.localName
-          )
-        : [];
+      const siblings = [...(at.parentElement?.children ?? [])].filter(
+        (c) => c.localName === at.localName
+      );
       const nth =
         siblings.length > 1 ? `:nth-of-type(${siblings.indexOf(at) + 1})` : '';
       steps.unshift(at.localName + nth);
@@ -399,6 +428,23 @@ export function installMutationRecorderInPage(options) {
     };
   };
 
+  // Mutation targets identify an existing node. Only additions need markup.
+  const identify = (node) => {
+    if (!node || isIgnored(node)) {
+      return null;
+    }
+    const element = node.nodeType === 1 ? node : node.parentElement;
+    return {
+      type: node.nodeType === 3 ? 'text' : 'element',
+      path: pathOf(element),
+      tag: element?.localName,
+      id: element?.id || null,
+      visible:
+        Boolean(element?.getClientRects().length) &&
+        window.getComputedStyle(element).visibility !== 'hidden',
+    };
+  };
+
   /**
    * Describe a node that is no longer where it was.
    *
@@ -458,13 +504,15 @@ export function installMutationRecorderInPage(options) {
         .map((record) => {
           const entry = {
             kind: record.type,
-            target: describe(record.target),
+            target: identify(record.target),
           };
           if (record.type === 'attributes') {
             entry.attribute = record.attributeName;
             const secret =
               isSecret(record.target) ||
-              redactSelectors.includes(record.attributeName);
+              redactAttributes.includes(record.attributeName) ||
+              (useDefaults &&
+                /token|csrf|xsrf|secret|password/i.test(record.attributeName));
             entry.before = secret ? redacted : record.oldValue;
             entry.after =
               secret || !record.target.getAttribute
@@ -485,8 +533,8 @@ export function installMutationRecorderInPage(options) {
             entry.removed = [...record.removedNodes]
               .filter((node) => !isIgnored(node))
               .map((node) => describeAt(node, record));
-            entry.previous = describe(record.previousSibling);
-            entry.next = describe(record.nextSibling);
+            entry.previous = identify(record.previousSibling);
+            entry.next = identify(record.nextSibling);
           }
           return entry;
         }),
@@ -511,7 +559,6 @@ export function installMutationRecorderInPage(options) {
     // into a field and then leaving it fires `input` for each keystroke and
     // `change` for the value it ended at, and a checkbox fires both for the
     // one click. Replaying a state that is already on screen shows nothing, so
-    // a repeat of what this element last reported is not recorded.
     const repeats = (element, property, after) => {
       const seen = lastLive.get(element) || {};
       const asText = JSON.stringify(after === undefined ? null : after);
@@ -526,16 +573,13 @@ export function installMutationRecorderInPage(options) {
     const liveEntry = (target, property, after, before) => ({
       kind: 'live-state',
       property,
-      target: describe(target),
+      target: identify(target),
       before: before === undefined ? null : before,
       after,
     });
 
     const pushLive = (entry) => {
-      // Scrolling fires far more often than it changes anything worth
-      // replaying, so consecutive scrolls of one element collapse into the
-      // position it ended at. Every other property keeps each step, because
-      // "stepwise replay must show each state" (issue #93).
+      // Consecutive scrolls collapse; other state changes retain each step.
       const last = state.queue[state.queue.length - 1];
       const only = last && last.records.length === 1 ? last.records[0] : null;
       if (
@@ -608,7 +652,7 @@ export function installMutationRecorderInPage(options) {
       pushLive({
         kind: 'live-state',
         property: 'scroll',
-        target: scrolling ? describe(scrolling) : null,
+        target: scrolling ? identify(scrolling) : null,
         before: null,
         after: scrolling
           ? { top: scrolling.scrollTop, left: scrolling.scrollLeft }

package/src/traces/recorder.js

@@ -1,5 +1,6 @@
 import { abortTraceStart } from './start-cleanup.js';
 import { createTraceStop } from './stop.js';
+import { openRollingBundle } from './rolling-bundle.js';
 /**
  * The trace recorder (issue #87).
  *
@@ -166,13 +167,13 @@ export async function startTrace(options = {}) {
   // readable export of everything that had happened. The bundle stays
   // authoritative: nothing is written here that is not written there first.
   let linksSink = null;
-  const bundle = await openTraceBundle({
+  const bundle = await openRecorderBundle(options, {
     output,
     limits,
     strict,
     now,
     monotonic,
-    onEvent: (event) => linksSink?.event(event),
+    onEvent: (event, source) => linksSink?.event(event, source),
   });
   const startedAt = new Date(now()).toISOString();
 
@@ -297,7 +298,13 @@ export async function startTrace(options = {}) {
       identity,
       now,
     });
-    detachNetwork = attachNetwork({ page, network, record, note });
+    detachNetwork = attachNetwork({
+      page,
+      network,
+      record,
+      note,
+      privacy: privacyOptions,
+    });
     video = options.video
       ? await startRecording({
           page,
@@ -405,6 +412,12 @@ export async function startTrace(options = {}) {
 
     return {
       path: bundle.root,
+      get currentPath() {
+        return bundle.currentPath ?? bundle.root;
+      },
+      get segments() {
+        return bundle.segments;
+      },
       links: linksSink?.path ?? null,
       mode: recorderMode,
       get bytesWritten() {
@@ -449,3 +462,20 @@ function requireTraceSupport(options) {
   validateNetworkOptions(options.network);
   (options.page ?? options.commander?.page)?.requireFeature?.('tracing');
 }
+
+function openRecorderBundle(options, settings) {
+  const writer =
+    options.openBundle ??
+    (options.mode === TRACE_MODE.CONTINUOUS
+      ? (config) =>
+          openRollingBundle(
+            config,
+            {
+              lazy: true,
+              maxBytes: settings.limits.maxBundleBytes ?? 256 * 1024 * 1024,
+            },
+            options
+          )
+      : openTraceBundle);
+  return writer(settings);
+}

package/src/traces/redaction.js

@@ -13,6 +13,7 @@ export const REDACTED = '[redacted]';
 /** Controls whose value is a secret unless the caller says otherwise. */
 export const DEFAULT_REDACT_SELECTORS = Object.freeze([
   'input[type=password]',
+  'input[type=hidden]',
   '[data-private]',
   '[data-bc-redact]',
 ]);
@@ -42,8 +43,74 @@ export const DEFAULT_REDACT_QUERY_PARAMS = Object.freeze([
   'session',
   'signature',
   'token',
+  'csrf',
+  'xsrf',
+  '_xsrf',
 ]);
 
+/** Match names rather than values, including framework-specific token names. */
+export function sensitiveName(name, privacy) {
+  return (
+    Boolean(privacy.useDefaults) &&
+    /password|passwd|token|csrf|xsrf|otp|secret|api[_-]?key|authorization/i.test(
+      name
+    )
+  );
+}
+
+function redactFields(value, privacy) {
+  if (Array.isArray(value)) {
+    return value.map((entry) => redactFields(entry, privacy));
+  }
+  if (value && typeof value === 'object') {
+    return Object.fromEntries(
+      Object.entries(value).map(([key, entry]) => [
+        key,
+        sensitiveName(key, privacy) ? REDACTED : redactFields(entry, privacy),
+      ])
+    );
+  }
+  return value;
+}
+
+function redactStructuredText(value, privacy) {
+  if (!privacy.useDefaults) {
+    return value;
+  }
+  try {
+    const parsed = JSON.parse(value);
+    if (parsed && typeof parsed === 'object') {
+      return JSON.stringify(redactFields(parsed, privacy));
+    }
+  } catch {
+    /* HTML, forms and script assignments are handled below. */
+  }
+  return value
+    .replace(
+      /(\b(?:["']?[\w-]+["']?)\s*[:=]\s*)("(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\d+(?:\.\d+)?)/g,
+      (match, prefix) =>
+        sensitiveName(prefix, privacy)
+          ? `${prefix}${JSON.stringify(REDACTED)}`
+          : match
+    )
+    .replace(/(^|[&?])([^=&\s]+)=([^&\s]*)/g, (match, start, key) => {
+      let name = key;
+      try {
+        name = decodeURIComponent(key.replaceAll('+', ' '));
+      } catch {
+        /* Keep malformed names. */
+      }
+      return sensitiveName(name, privacy)
+        ? `${start}${key}=${REDACTED}`
+        : match;
+    })
+    .replace(
+      /(Content-Disposition:[^\r\n]*?;\s*name="([^"]+)"[^\r\n]*\r?\n(?:[^\r\n]+\r?\n)*\r?\n)([\s\S]*?)(?=\r?\n--)/gi,
+      (match, header, name) =>
+        sensitiveName(name, privacy) ? `${header}${REDACTED}` : match
+    );
+}
+
 function lowerSet(values) {
   return [...new Set(values.map((value) => String(value).toLowerCase()))];
 }
@@ -75,6 +142,7 @@ export function normalizePrivacyOptions(privacy = {}) {
   }
 
   return {
+    useDefaults,
     redactSelectors: lowerSet([
       ...(useDefaults ? DEFAULT_REDACT_SELECTORS : []),
       ...redactSelectors,
@@ -107,7 +175,8 @@ export function redactText(value, privacy, context = {}) {
     return value;
   }
 
-  let text = value;
+  let text =
+    context.kind === 'url' ? value : redactStructuredText(value, privacy);
   for (const pattern of privacy.redactPatterns || []) {
     // A caller's regex may be stateful; resetting keeps one trace's output
     // from depending on the previous record's match position.
@@ -148,7 +217,10 @@ export function redactUrl(url, privacy) {
       parsed.password = parsed.password ? REDACTED : '';
     }
     for (const key of [...parsed.searchParams.keys()]) {
-      if (privacy.redactQueryParams.includes(key.toLowerCase())) {
+      if (
+        privacy.redactQueryParams.includes(key.toLowerCase()) ||
+        sensitiveName(key, privacy)
+      ) {
         parsed.searchParams.set(key, REDACTED);
       }
     }
@@ -157,7 +229,10 @@ export function redactUrl(url, privacy) {
       const fragment = new URLSearchParams(parsed.hash.slice(1));
       let changed = false;
       for (const key of [...fragment.keys()]) {
-        if (privacy.redactQueryParams.includes(key.toLowerCase())) {
+        if (
+          privacy.redactQueryParams.includes(key.toLowerCase()) ||
+          sensitiveName(key, privacy)
+        ) {
           fragment.set(key, REDACTED);
           changed = true;
         }
@@ -190,9 +265,11 @@ export function redactHeaders(headers, privacy) {
 
   const result = {};
   for (const [name, value] of Object.entries(headers)) {
-    result[name] = privacy.redactAttributes.includes(name.toLowerCase())
-      ? REDACTED
-      : redactText(String(value), privacy, { kind: 'header', name });
+    result[name] =
+      privacy.redactAttributes.includes(name.toLowerCase()) ||
+      sensitiveName(name, privacy)
+        ? REDACTED
+        : redactText(String(value), privacy, { kind: 'header', name });
   }
   return result;
 }
@@ -209,6 +286,12 @@ export function redactHeaders(headers, privacy) {
  * @returns {*} The value with strings redacted
  */
 export function redactValue(value, privacy, key = '') {
+  if (
+    sensitiveName(key, privacy) ||
+    privacy.redactAttributes.includes(key.toLowerCase())
+  ) {
+    return REDACTED;
+  }
   if (typeof value === 'string') {
     if (privacy.redactAttributes.includes(key.toLowerCase())) {
       return REDACTED;