browser-commander 0.27.0 → 0.29.0

package/src/traces/redaction.js

@@ -13,6 +13,7 @@ export const REDACTED = '[redacted]';
 /** Controls whose value is a secret unless the caller says otherwise. */
 export const DEFAULT_REDACT_SELECTORS = Object.freeze([
   'input[type=password]',
+  'input[type=hidden]',
   '[data-private]',
   '[data-bc-redact]',
 ]);
@@ -42,8 +43,74 @@ export const DEFAULT_REDACT_QUERY_PARAMS = Object.freeze([
   'session',
   'signature',
   'token',
+  'csrf',
+  'xsrf',
+  '_xsrf',
 ]);
 
+/** Match names rather than values, including framework-specific token names. */
+export function sensitiveName(name, privacy) {
+  return (
+    Boolean(privacy.useDefaults) &&
+    /password|passwd|token|csrf|xsrf|otp|secret|api[_-]?key|authorization/i.test(
+      name
+    )
+  );
+}
+
+function redactFields(value, privacy) {
+  if (Array.isArray(value)) {
+    return value.map((entry) => redactFields(entry, privacy));
+  }
+  if (value && typeof value === 'object') {
+    return Object.fromEntries(
+      Object.entries(value).map(([key, entry]) => [
+        key,
+        sensitiveName(key, privacy) ? REDACTED : redactFields(entry, privacy),
+      ])
+    );
+  }
+  return value;
+}
+
+function redactStructuredText(value, privacy) {
+  if (!privacy.useDefaults) {
+    return value;
+  }
+  try {
+    const parsed = JSON.parse(value);
+    if (parsed && typeof parsed === 'object') {
+      return JSON.stringify(redactFields(parsed, privacy));
+    }
+  } catch {
+    /* HTML, forms and script assignments are handled below. */
+  }
+  return value
+    .replace(
+      /(\b(?:["']?[\w-]+["']?)\s*[:=]\s*)("(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\d+(?:\.\d+)?)/g,
+      (match, prefix) =>
+        sensitiveName(prefix, privacy)
+          ? `${prefix}${JSON.stringify(REDACTED)}`
+          : match
+    )
+    .replace(/(^|[&?])([^=&\s]+)=([^&\s]*)/g, (match, start, key) => {
+      let name = key;
+      try {
+        name = decodeURIComponent(key.replaceAll('+', ' '));
+      } catch {
+        /* Keep malformed names. */
+      }
+      return sensitiveName(name, privacy)
+        ? `${start}${key}=${REDACTED}`
+        : match;
+    })
+    .replace(
+      /(Content-Disposition:[^\r\n]*?;\s*name="([^"]+)"[^\r\n]*\r?\n(?:[^\r\n]+\r?\n)*\r?\n)([\s\S]*?)(?=\r?\n--)/gi,
+      (match, header, name) =>
+        sensitiveName(name, privacy) ? `${header}${REDACTED}` : match
+    );
+}
+
 function lowerSet(values) {
   return [...new Set(values.map((value) => String(value).toLowerCase()))];
 }
@@ -75,6 +142,7 @@ export function normalizePrivacyOptions(privacy = {}) {
   }
 
   return {
+    useDefaults,
     redactSelectors: lowerSet([
       ...(useDefaults ? DEFAULT_REDACT_SELECTORS : []),
       ...redactSelectors,
@@ -107,7 +175,8 @@ export function redactText(value, privacy, context = {}) {
     return value;
   }
 
-  let text = value;
+  let text =
+    context.kind === 'url' ? value : redactStructuredText(value, privacy);
   for (const pattern of privacy.redactPatterns || []) {
     // A caller's regex may be stateful; resetting keeps one trace's output
     // from depending on the previous record's match position.
@@ -148,7 +217,10 @@ export function redactUrl(url, privacy) {
       parsed.password = parsed.password ? REDACTED : '';
     }
     for (const key of [...parsed.searchParams.keys()]) {
-      if (privacy.redactQueryParams.includes(key.toLowerCase())) {
+      if (
+        privacy.redactQueryParams.includes(key.toLowerCase()) ||
+        sensitiveName(key, privacy)
+      ) {
         parsed.searchParams.set(key, REDACTED);
       }
     }
@@ -157,7 +229,10 @@ export function redactUrl(url, privacy) {
       const fragment = new URLSearchParams(parsed.hash.slice(1));
       let changed = false;
       for (const key of [...fragment.keys()]) {
-        if (privacy.redactQueryParams.includes(key.toLowerCase())) {
+        if (
+          privacy.redactQueryParams.includes(key.toLowerCase()) ||
+          sensitiveName(key, privacy)
+        ) {
           fragment.set(key, REDACTED);
           changed = true;
         }
@@ -190,9 +265,11 @@ export function redactHeaders(headers, privacy) {
 
   const result = {};
   for (const [name, value] of Object.entries(headers)) {
-    result[name] = privacy.redactAttributes.includes(name.toLowerCase())
-      ? REDACTED
-      : redactText(String(value), privacy, { kind: 'header', name });
+    result[name] =
+      privacy.redactAttributes.includes(name.toLowerCase()) ||
+      sensitiveName(name, privacy)
+        ? REDACTED
+        : redactText(String(value), privacy, { kind: 'header', name });
   }
   return result;
 }
@@ -209,6 +286,12 @@ export function redactHeaders(headers, privacy) {
  * @returns {*} The value with strings redacted
  */
 export function redactValue(value, privacy, key = '') {
+  if (
+    sensitiveName(key, privacy) ||
+    privacy.redactAttributes.includes(key.toLowerCase())
+  ) {
+    return REDACTED;
+  }
   if (typeof value === 'string') {
     if (privacy.redactAttributes.includes(key.toLowerCase())) {
       return REDACTED;

package/src/traces/render.js

@@ -0,0 +1,168 @@
+import { writeCapture } from '../capture/files.js';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { readTrace } from './reader.js';
+import {
+  encodeAnimation,
+  decodePng,
+  encodePng,
+  resizeFrame,
+} from '../capture/encoding.js';
+import { encodeVideo } from '../capture/video.js';
+
+export function withinTime(record, options = {}) {
+  const time = (value) =>
+    typeof value === 'number' || /^\d+(\.\d+)?$/.test(value)
+      ? { offset: Number(value) * 1000 }
+      : {
+          date: Date.parse(
+            /^\d{2}:\d{2}(?::\d{2})?$/.test(value)
+              ? `${options.startedAt?.slice(0, 10)}T${value.length === 5 ? `${value}:00` : value}Z`
+              : value
+          ),
+        };
+  for (const [name, direction] of [
+    ['from', -1],
+    ['to', 1],
+  ]) {
+    if (options[name] === undefined) {
+      continue;
+    }
+    const bound = time(options[name]);
+    if (bound.date !== undefined && !Number.isFinite(bound.date)) {
+      throw new RangeError(`invalid ${name} timestamp`);
+    }
+    const value =
+      bound.offset !== undefined
+        ? (record.monotonicMs ?? 0) - (options.monotonicOrigin ?? 0)
+        : Date.parse(record.at);
+    if (
+      direction === -1
+        ? value < (bound.offset ?? bound.date)
+        : value > (bound.offset ?? bound.date)
+    ) {
+      return false;
+    }
+  }
+  return true;
+}
+export async function summarizeTrace(root, options = {}) {
+  const opened = await readTrace(root);
+  options = timelineOptions(opened, options);
+  const records = [...opened.events];
+  const seenMutations = new Set();
+  for (const event of opened.events.filter(
+    (event) => event.kind === 'mutations'
+  )) {
+    const key = `${event.segment ?? ''}:${event.checkpoint}`;
+    if (seenMutations.has(key)) {
+      continue;
+    }
+    seenMutations.add(key);
+    for (const batch of await opened.mutations(event.checkpoint)) {
+      for (const mutation of batch.records ?? []) {
+        records.push({
+          ...mutation,
+          kind: `dom.${mutation.kind}`,
+          at: new Date(batch.at).toISOString(),
+          monotonicMs: event.monotonicMs,
+        });
+      }
+    }
+  }
+  return records
+    .filter(
+      (record) =>
+        withinTime(record, options) &&
+        (!options.grep || JSON.stringify(record).includes(options.grep))
+    )
+    .sort(
+      (a, b) =>
+        Date.parse(a.at) - Date.parse(b.at) ||
+        (a.sequence ?? 0) - (b.sequence ?? 0)
+    );
+}
+export async function renderTrace(root, options = {}) {
+  const opened = await readTrace(root);
+  options = timelineOptions(opened, options);
+  const frames = [];
+  let compressedBytes = 0;
+  for (const event of opened.events.filter(
+    (event) => event.kind === 'checkpoint' && withinTime(event, options)
+  )) {
+    const member = event.members?.screenshot;
+    if (member) {
+      if (frames.length >= 1000) {
+        throw new RangeError('trace render exceeds 1000 frames');
+      }
+      const file = path.join(opened.path, member);
+      compressedBytes += (await fs.stat(file)).size;
+      if (compressedBytes > 64 * 1024 * 1024) {
+        throw new RangeError('trace render frames exceed 64 MiB');
+      }
+      frames.push(await fs.readFile(file));
+    }
+  }
+  if (frames.length === 0) {
+    throw new Error('trace has no checkpoint screenshots in this interval');
+  }
+  if (frames.length > 1000) {
+    throw new RangeError('trace render exceeds 1000 frames');
+  }
+  const format =
+    options.format ?? (path.extname(options.path ?? '').slice(1) || 'gif');
+  if (format === 'sheet') {
+    let total = 0;
+    const images = frames.map((frame) => {
+      if (frame.length < 33) {
+        throw new RangeError('frame must be a PNG');
+      }
+      total +=
+        frame.readUInt32BE(16) *
+        frame.readUInt32BE(20) *
+        4 *
+        Math.max(1, (options.scale ?? 1) ** 2);
+      if (total > 256 * 1024 * 1024) {
+        throw new RangeError('contact sheet frames exceed 256 MiB');
+      }
+      return resizeFrame(decodePng(frame), options.scale ?? 1);
+    });
+    const columns = options.columns ?? Math.min(4, images.length);
+    if (!Number.isInteger(columns) || columns < 1 || columns > 1000) {
+      throw new RangeError('columns must be 1–1000');
+    }
+    const width = Math.max(...images.map((image) => image.width)) * columns;
+    const rowHeight = Math.max(...images.map((image) => image.height));
+    const height = rowHeight * Math.ceil(images.length / columns);
+    if (width * height > 16_777_216) {
+      throw new RangeError('contact sheet exceeds 16 megapixels');
+    }
+    const data = new Uint8Array(width * height * 4);
+    images.forEach((image, index) => {
+      const x = (index % columns) * (width / columns),
+        y = Math.floor(index / columns) * rowHeight;
+      for (let row = 0; row < image.height; row++) {
+        data.set(
+          image.data.subarray(
+            row * image.width * 4,
+            (row + 1) * image.width * 4
+          ),
+          ((y + row) * width + x) * 4
+        );
+      }
+    });
+    const bytes = encodePng({ width, height, data });
+    return writeCapture(bytes, options.path);
+  }
+  return ['mp4', 'webm', 'mov'].includes(format)
+    ? encodeVideo(frames, { ...options, format })
+    : encodeAnimation(frames, { ...options, format });
+}
+
+function timelineOptions(opened, options) {
+  return {
+    ...options,
+    monotonicOrigin: opened.events[0]?.monotonicMs ?? 0,
+    startedAt: opened.events[0]?.at,
+  };
+}

package/src/traces/rolling-bundle.js

@@ -0,0 +1,240 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { openTraceBundle } from './bundle.js';
+import { createManifest } from './schema.js';
+
+/** Rotate complete bundles at the producer's write boundary, including background events. */
+export async function openRollingBundle(options, config, traceOptions) {
+  const root = path.resolve(options.output);
+  const maxBytes =
+    config.maxBytes ?? options.limits.maxBundleBytes ?? 32 * 1024 * 1024;
+  const maxSegments = config.maxSegments ?? Infinity;
+  if (
+    !Number.isSafeInteger(maxBytes) ||
+    maxBytes < 1024 ||
+    (maxSegments !== Infinity &&
+      (!Number.isSafeInteger(maxSegments) ||
+        maxSegments < 1 ||
+        maxSegments > 100))
+  ) {
+    throw new RangeError('invalid trace rotation bounds');
+  }
+  let current,
+    sequence = 0,
+    pinned = false,
+    base = null,
+    baseEvent = null;
+  let eventSequence = 0;
+  const segments = [],
+    writers = [];
+  let queue = Promise.resolve();
+  const run = (fn) => {
+    const result = queue.then(fn);
+    queue = result.catch(() => {});
+    return result;
+  };
+  const template = () =>
+    createManifest({
+      mode: traceOptions.mode ?? 'checkpoints',
+      engine: traceOptions.commander?.engine,
+      startedAt: new Date().toISOString(),
+      stoppedAt: new Date().toISOString(),
+      dom: traceOptions.dom,
+      limits: traceOptions.limits,
+    });
+  async function open() {
+    const name = `segment-${String(++sequence).padStart(6, '0')}`;
+    current = await openTraceBundle({
+      ...options,
+      output: path.join(root, name),
+      nextSequence: () => ++eventSequence,
+      limits: { ...options.limits, maxBundleBytes: Infinity },
+      onEvent: (event) => options.onEvent?.(event, { root: current.root }),
+    });
+    writers.push(current);
+    segments.push(name);
+    if (segments.length > maxSegments) {
+      await fs.rm(path.join(root, segments.shift()), {
+        recursive: true,
+        force: true,
+      });
+    }
+    await fs.writeFile(
+      path.join(root, 'segments.json.tmp'),
+      JSON.stringify({ segments }),
+      { mode: 0o600 }
+    );
+    await fs.rename(
+      path.join(root, 'segments.json.tmp'),
+      path.join(root, 'segments.json')
+    );
+  }
+  async function prepare(bytes, carry = false) {
+    if (
+      !pinned &&
+      current.bytesWritten > 0 &&
+      current.bytesWritten + bytes > maxBytes
+    ) {
+      await current.close(template());
+      if (sequence === 0) {
+        const name = 'segment-000001';
+        const destination = path.join(root, name);
+        await fs.mkdir(destination, { mode: 0o700 });
+        for (const member of [
+          'events.ndjson',
+          'manifest.json',
+          'checkpoints',
+          'mutations',
+          'artifacts',
+          'viewer.html',
+        ]) {
+          await fs
+            .rename(path.join(root, member), path.join(destination, member))
+            .catch((error) => {
+              if (error.code !== 'ENOENT') {
+                throw error;
+              }
+            });
+        }
+        sequence = 1;
+        segments.push(name);
+      }
+      await open();
+      if (carry && base && baseEvent) {
+        const members = await current.writeCheckpoint(base);
+        await current.appendEvent({
+          ...baseEvent,
+          members,
+          reason: 'rotation-base',
+        });
+      }
+    }
+  }
+  await fs.mkdir(root, { recursive: true, mode: 0o700 });
+  if (config.lazy) {
+    current = await openTraceBundle({
+      ...options,
+      nextSequence: () => ++eventSequence,
+      limits: { ...options.limits, maxBundleBytes: Infinity },
+    });
+    writers.push(current);
+  } else {
+    await open();
+  }
+  const counts = () =>
+    writers.reduce(
+      (sum, b) => {
+        for (const k of Object.keys(sum)) {
+          sum[k] += b.counts[k];
+        }
+        return sum;
+      },
+      { checkpoints: 0, events: 0, mutationBatches: 0 }
+    );
+  return {
+    root,
+    get currentPath() {
+      return current.root;
+    },
+    get segments() {
+      return [...segments];
+    },
+    get counts() {
+      return counts();
+    },
+    get problems() {
+      return writers.flatMap((b) => b.problems);
+    },
+    get dropped() {
+      return writers.reduce((sum, b) => sum + b.dropped, 0);
+    },
+    get bytesWritten() {
+      return current.bytesWritten;
+    },
+    get mutationTruncated() {
+      return current.mutationTruncated;
+    },
+    appendEvent: (event) =>
+      run(async () => {
+        await prepare(Buffer.byteLength(JSON.stringify(event)) + 256, true);
+        const result = await current.appendEvent(event);
+        if (event.kind === 'checkpoint') {
+          baseEvent = result;
+        }
+        if (
+          event.kind === 'checkpoint' ||
+          (event.kind === 'mutations' && pinned === 'mutations')
+        ) {
+          pinned = false;
+        }
+        return result;
+      }),
+    writeCheckpoint: (checkpoint) =>
+      run(async () => {
+        const bytes =
+          Buffer.byteLength(checkpoint.html ?? '') +
+          Buffer.byteLength(JSON.stringify(checkpoint.state ?? {})) +
+          (checkpoint.screenshot?.length ?? 0) +
+          1024;
+        await prepare(bytes);
+        pinned = 'checkpoint';
+        base = checkpoint;
+        return current.writeCheckpoint(checkpoint);
+      }),
+    writeMutations: (index, batches) =>
+      run(async () => {
+        await prepare(Buffer.byteLength(JSON.stringify(batches)) + 256, true);
+        const result = await current.writeMutations(index, batches);
+        if (pinned !== 'checkpoint') {
+          pinned = result ? 'mutations' : false;
+        }
+        return result;
+      }),
+    writeMember: (member, contents, writeOptions) =>
+      run(async () => {
+        await prepare(Buffer.byteLength(contents));
+        return current.writeMember(member, contents, writeOptions);
+      }),
+    writeArtifact: (contents, extension) =>
+      run(async () => {
+        await prepare(Buffer.byteLength(contents));
+        const result = await current.writeArtifact(contents, extension);
+        return result
+          ? {
+              ...result,
+              member: segments.length
+                ? `${segments.at(-1)}/${result.member}`
+                : result.member,
+            }
+          : null;
+      }),
+    drop: (record) => run(() => current.drop(record)),
+    abort: () => run(() => current.abort()),
+    close: (manifest) =>
+      run(async () => {
+        manifest = await current.close(manifest);
+        if (writers.some((writer) => writer.dropped)) {
+          manifest.outcome = 'partial';
+        }
+        // Finished segments get the final capabilities and outcome as well.
+        for (const name of segments) {
+          const file = path.join(root, name, 'manifest.json');
+          const previous = JSON.parse(await fs.readFile(file, 'utf8'));
+          await fs.writeFile(
+            file,
+            JSON.stringify({
+              ...manifest,
+              counts: previous.counts,
+              dropped: previous.dropped,
+            }),
+            { mode: 0o600 }
+          );
+        }
+        return {
+          ...manifest,
+          counts: counts(),
+          dropped: writers.reduce((sum, b) => sum + b.dropped, 0),
+        };
+      }),
+  };
+}

package/src/traces/rolling.js

@@ -0,0 +1,77 @@
+import path from 'node:path';
+
+/** Keep the ordinary trace handle while routing every write through rotation. */
+export async function startRollingTrace(options, start) {
+  const { openRollingBundle } = await import('./rolling-bundle.js');
+  const config = options.limits.rotate === true ? {} : options.limits.rotate;
+  return start({
+    ...options,
+    limits: { ...options.limits, rotate: false },
+    openBundle: (bundleOptions) =>
+      openRollingBundle(bundleOptions, config, options),
+  });
+}
+
+/** Combine segment timelines while keeping checkpoint member paths distinct. */
+export async function readRollingTrace(root, index, read) {
+  const events = [],
+    checkpoints = [],
+    locations = new Map();
+  let manifest,
+    truncated = false,
+    number = 0;
+  for (const segment of index.segments) {
+    if (!/^segment-\d+$/.test(segment)) {
+      throw new Error('invalid trace segment path');
+    }
+    const opened = await read(path.join(root, segment));
+    manifest = opened.manifest;
+    truncated ||= opened.truncated;
+    const ids = new Map();
+    for (const checkpoint of opened.checkpoints) {
+      const id = ++number;
+      ids.set(checkpoint.index, id);
+      locations.set(id, { opened, index: checkpoint.index });
+      checkpoints.push({
+        ...checkpoint,
+        index: id,
+        members: Object.fromEntries(
+          Object.entries(checkpoint.members).map(([key, value]) => [
+            key,
+            `${segment}/${value}`,
+          ])
+        ),
+      });
+    }
+    for (const event of opened.events) {
+      const mapped = { ...event, segment };
+      if (event.kind === 'checkpoint') {
+        Object.assign(
+          mapped,
+          checkpoints.find(
+            (checkpoint) => checkpoint.index === ids.get(event.index)
+          )
+        );
+      }
+      if (event.kind === 'mutations') {
+        mapped.checkpoint = ids.get(event.checkpoint) ?? 0;
+      }
+      events.push(mapped);
+    }
+  }
+  return {
+    path: root,
+    manifest,
+    events,
+    checkpoints,
+    truncated,
+    html: (index) =>
+      locations.get(index)?.opened.html(locations.get(index).index),
+    state: (index) =>
+      locations.get(index)?.opened.state(locations.get(index).index),
+    screenshot: (index) =>
+      locations.get(index)?.opened.screenshot(locations.get(index).index),
+    mutations: (index) =>
+      locations.get(index)?.opened.mutations(locations.get(index).index) ?? [],
+  };
+}

package/src/traces/start-cleanup.js

@@ -0,0 +1,15 @@
+/** Release resources when startup fails before returning a running trace. */
+export async function abortTraceStart(resources) {
+  const { detachers, detachNetwork, video, mutations, linksSink, bundle } =
+    resources;
+  for (const detach of detachers.reverse()) {
+    await Promise.resolve()
+      .then(detach)
+      .catch(() => {});
+  }
+  await detachNetwork().catch(() => {});
+  await video?.stop().catch(() => {});
+  await mutations.stop().catch(() => {});
+  await linksSink?.discard().catch(() => {});
+  await bundle.abort();
+}