browser-commander 0.27.0 → 0.29.0

package/src/traces/links.js

@@ -429,6 +429,7 @@ export async function traceLinks(trace, options = {}) {
   if (sections.has('control-diffs')) {
     links.push(...(await controlDiffLinks(opened)));
   }
+  links.push(...(await domLinks(opened, options.dom)));
 
   if (sections.has('trace')) {
     links.push(
@@ -470,6 +471,63 @@ async function controlDiffLinks(opened) {
   return links;
 }
 
+/** Self-contained DOM and mutation records in Links Notation. */
+async function domLinks(opened, dom, seen = new Set()) {
+  if (!dom) {
+    return [];
+  }
+  const links = [];
+  for (const checkpoint of opened.checkpoints) {
+    if (dom === 'text') {
+      const state = await opened.state(checkpoint.index);
+      if (state?.text) {
+        links.push(
+          new Link('dom-text', [
+            field('checkpoint', checkpoint.index),
+            field('text', state.text),
+          ])
+        );
+      }
+    } else {
+      links.push(
+        new Link('dom-snapshot', [
+          field('checkpoint', checkpoint.index),
+          field('html', await opened.html(checkpoint.index)),
+        ])
+      );
+    }
+  }
+  for (const event of opened.events.filter(
+    (event) => event.kind === 'mutations'
+  )) {
+    const batches = (await opened.mutations(event.checkpoint)).filter(
+      (batch) => {
+        const key = `${event.member}:${batch.frameId}:${batch.sequence}:${batch.at}`;
+        if (seen.has(key)) {
+          return false;
+        }
+        seen.add(key);
+        return true;
+      }
+    );
+    for (const batch of batches) {
+      for (const record of batch.records ?? []) {
+        if (dom !== 'text') {
+          links.push(
+            new Link('dom-mutation', [
+              field('at', batch.at),
+              field('record', JSON.stringify(record)),
+            ])
+          );
+        } else {
+          links.push(...textMutationLinks(record, batch));
+        }
+      }
+    }
+  }
+  return links;
+}
+
 /**
  * Resolve where an export should be written.
  *
@@ -521,10 +579,13 @@ export async function writeTraceLinks(trace, output, options = {}) {
  * @returns {Promise<Object>} The sink
  */
 export async function openTraceLinks(options = {}) {
-  const { output, include, bundlePath, ...about } = options;
+  const { output, include, dom, bundlePath, ...about } = options;
   const file = await resolveOutput(output);
   const sections = chosenSections(include);
   const problems = [];
+  // Only the current interval is remembered; completed intervals never recur.
+  let mutationMember = null;
+  let mutationOffset = 0;
 
   await fs.mkdir(path.dirname(file), { recursive: true });
   const handle = await fs.open(file, 'w', 0o600);
@@ -567,11 +628,66 @@ export async function openTraceLinks(options = {}) {
      * @param {Object} event - The event as written to the bundle
      * @returns {Promise<void>}
      */
-    async event(event) {
+    async event(event, source = {}) {
       if (!event) {
         return;
       }
       await append(linksForEvent(event, sections));
+      if (
+        dom &&
+        bundlePath &&
+        [TRACE_EVENT.CHECKPOINT, TRACE_EVENT.MUTATIONS].includes(event.kind)
+      ) {
+        const root = source.root ?? bundlePath;
+        let batches = [];
+        if (event.kind === TRACE_EVENT.MUTATIONS) {
+          const member = path.join(root, event.member);
+          if (member !== mutationMember) {
+            mutationMember = member;
+            mutationOffset = 0;
+          }
+          const input = await fs.open(member, 'r');
+          try {
+            const size = (await input.stat()).size;
+            const bytes = Buffer.alloc(Math.max(0, size - mutationOffset));
+            await input.read(bytes, 0, bytes.length, mutationOffset);
+            mutationOffset = size;
+            batches = bytes
+              .toString('utf8')
+              .split('\n')
+              .filter(Boolean)
+              .map(JSON.parse);
+          } finally {
+            await input.close();
+          }
+        }
+        const opened = {
+          html: () =>
+            event.members?.html
+              ? fs.readFile(path.join(root, event.members.html), 'utf8')
+              : null,
+          state: async () =>
+            event.members?.state
+              ? JSON.parse(
+                  await fs.readFile(
+                    path.join(root, event.members.state),
+                    'utf8'
+                  )
+                )
+              : null,
+          mutations: () => batches,
+        };
+        await append(
+          await domLinks(
+            {
+              ...opened,
+              checkpoints: event.kind === TRACE_EVENT.CHECKPOINT ? [event] : [],
+              events: event.kind === TRACE_EVENT.MUTATIONS ? [event] : [],
+            },
+            dom
+          )
+        );
+      }
     },
     /**
      * Finish the export.
@@ -593,6 +709,7 @@ export async function openTraceLinks(options = {}) {
           problems.push({ member: file, detail: error.message });
         }
       }
+
       if (sections.has('trace')) {
         links.push(
           traceResultLink(closing.manifest, {
@@ -621,3 +738,41 @@ export async function openTraceLinks(options = {}) {
     },
   };
 }
+
+function textMutationLinks(record, batch) {
+  if (record.target?.visible === false) {
+    return [];
+  }
+  const links = [];
+  const target = record.target?.path ?? '';
+  if (record.kind === 'attributes' || record.kind === 'characterData') {
+    links.push(
+      new Link(
+        record.kind === 'attributes' ? 'attribute-changed' : 'text-changed',
+        [
+          field('at', batch.at),
+          field('path', target),
+          field('attribute', record.attribute ?? null),
+          field('before', record.before),
+          field('after', record.after),
+        ]
+      )
+    );
+  }
+  if (record.kind === 'childList') {
+    for (const change of ['added', 'removed']) {
+      for (const node of record[change] ?? []) {
+        if (node.visible !== false && node.text) {
+          links.push(
+            new Link(`text-${change}`, [
+              field('at', batch.at),
+              field('path', node.path ?? target),
+              field('text', node.text),
+            ])
+          );
+        }
+      }
+    }
+  }
+  return links;
+}

package/src/traces/mutation-stream.js

@@ -20,7 +20,7 @@ import {
   RECORDER_GLOBAL,
   stopMutationRecorderInPage,
 } from './page-capture.js';
-import { REDACTED } from './redaction.js';
+import { redactValue, REDACTED } from './redaction.js';
 import { TRACE_DROP_REASON, TRACE_EVENT } from './schema.js';
 
 /** In-page records kept before the recorder starts counting drops instead. */
@@ -62,7 +62,10 @@ export function createMutationStream(options) {
   const recorderOptions = {
     globalName: RECORDER_GLOBAL,
     redactSelectors: privacyOptions.redactSelectors,
+    redactAttributes: privacyOptions.redactAttributes,
+    useDefaults: privacyOptions.useDefaults,
     redacted: REDACTED,
+    ignoreSelectors: domOptions.ignoreSelectors ?? [],
     maxQueued: limits.maxQueuedMutations ?? DEFAULT_MAX_QUEUED_MUTATIONS,
     liveState: domOptions.liveState !== false,
   };
@@ -194,13 +197,17 @@ export function createMutationStream(options) {
       // frame; one ordered file is what a replay reads.
       batches.sort((left, right) => (left.at ?? 0) - (right.at ?? 0));
 
-      const member = await bundle.writeMutations(index, batches);
+      const member = await bundle.writeMutations(
+        index,
+        redactValue(batches, privacyOptions)
+      );
       if (member) {
         await record(TRACE_EVENT.MUTATIONS, {
           member,
           batches: batches.length,
           checkpoint: index,
           frames: drained.length,
+          truncated: bundle.mutationTruncated,
         });
       }
       return member;

package/src/traces/network.js

@@ -0,0 +1,309 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { withDeadline } from './deadline.js';
+import {
+  normalizePrivacyOptions,
+  redactText,
+  redactHeaders as privateHeaders,
+} from './redaction.js';
+
+const PRIVATE_HEADERS = new Set([
+  'cookie',
+  'authorization',
+  'proxy-authorization',
+  'set-cookie',
+]);
+export function redactHeaders(headers = {}) {
+  return privateHeaders(
+    Object.fromEntries(
+      Object.entries(headers).map(([key, value]) => [
+        key.toLowerCase(),
+        PRIVATE_HEADERS.has(key.toLowerCase()) ? '[redacted]' : value,
+      ])
+    ),
+    normalizePrivacyOptions()
+  );
+}
+
+export function isTextBody(contentType = '') {
+  return /^(?:text\/|application\/(?:[\w.+-]*json|[\w.+-]*xml|javascript|x-www-form-urlencoded))|multipart\/form-data/i.test(
+    contentType
+  );
+}
+
+function matches(url, pattern) {
+  if (pattern === undefined) {
+    return true;
+  }
+  if (typeof pattern === 'function') {
+    return Boolean(pattern(url));
+  }
+  if (pattern instanceof RegExp) {
+    pattern.lastIndex = 0;
+    return pattern.test(url);
+  }
+  return url.includes(String(pattern));
+}
+
+export function networkHar(events) {
+  const requests = new Map(
+    events
+      .filter((event) => event.kind === 'network.request')
+      .map((event) => [event.requestId, event])
+  );
+  const headers = (value) =>
+    Object.entries(value ?? {}).map(([name, value]) => ({
+      name,
+      value: String(value),
+    }));
+  const entries = events
+    .filter((event) => event.kind === 'network.response')
+    .map((response) => {
+      const request = requests.get(response.requestId) ?? response;
+      const timing = response.timing ?? request.timing ?? {};
+      return {
+        startedDateTime: request.at ?? new Date().toISOString(),
+        time: Math.max(0, timing.responseEnd ?? 0),
+        request: {
+          method: request.method,
+          url: request.url,
+          httpVersion: 'HTTP/1.1',
+          headers: headers(request.headers),
+          cookies: [],
+          queryString: [],
+          headersSize: -1,
+          bodySize: -1,
+          ...(request.postData !== undefined
+            ? {
+                postData: {
+                  mimeType: request.headers?.['content-type'] ?? '',
+                  text: request.postData,
+                  _truncated: request.postDataTruncated,
+                },
+              }
+            : {}),
+        },
+        response: {
+          status: response.status,
+          statusText: response.statusText ?? '',
+          httpVersion: 'HTTP/1.1',
+          headers: headers(response.headers),
+          cookies: [],
+          redirectURL: response.headers?.location ?? '',
+          headersSize: -1,
+          bodySize: response.body?.size ?? -1,
+          content: {
+            size: response.body?.size ?? 0,
+            mimeType: response.contentType ?? '',
+            ...(response.body?.data !== undefined
+              ? {
+                  text: response.body.data,
+                  ...(response.body.encoding === 'utf8'
+                    ? {}
+                    : { encoding: 'base64' }),
+                  _truncated: response.body.truncated,
+                }
+              : {}),
+          },
+        },
+        cache: {},
+        timings: {
+          send: 0,
+          wait: Math.max(
+            0,
+            (timing.responseStart ?? 0) - (timing.requestStart ?? 0)
+          ),
+          receive: Math.max(
+            0,
+            (timing.responseEnd ?? 0) - (timing.responseStart ?? 0)
+          ),
+        },
+        _resourceType: response.resourceType,
+      };
+    });
+  return {
+    log: {
+      version: '1.2',
+      creator: { name: 'browser-commander', version: '1' },
+      entries,
+    },
+  };
+}
+
+/** Observe without fetching again, and drain responses before closing the bundle. */
+export function validateNetworkOptions(network) {
+  if (!network) {
+    return;
+  }
+  const options = network === true ? {} : network;
+  if (
+    typeof options !== 'object' ||
+    !Number.isSafeInteger(options.maxBodyBytes ?? 1024 * 1024) ||
+    (options.maxBodyBytes ?? 0) < 0 ||
+    (options.maxBodyBytes ?? 0) > 16 * 1024 * 1024
+  ) {
+    throw new RangeError(
+      'network.maxBodyBytes must be an integer between 0 and 16 MiB'
+    );
+  }
+}
+export function attachNetwork({
+  page,
+  network,
+  record,
+  note,
+  privacy = normalizePrivacyOptions(),
+}) {
+  validateNetworkOptions(network);
+  if (!network) {
+    return async () => {};
+  }
+  const options = network === true ? {} : network;
+  const maxBodyBytes = options.maxBodyBytes ?? 1024 * 1024;
+  if (!Number.isSafeInteger(maxBodyBytes) || maxBodyBytes < 0) {
+    throw new RangeError('network.maxBodyBytes must be a nonnegative integer');
+  }
+  const ids = new WeakMap();
+  const pending = new Set();
+  let sequence = 0;
+  const details = (request) => ({
+    requestId: ids.get(request),
+    method: request.method(),
+    url: request.url(),
+    resourceType: request.resourceType?.() ?? 'other',
+    timing: request.timing?.() ?? null,
+  });
+  const allowed = (request) =>
+    (!options.resourceTypes ||
+      options.resourceTypes.includes(request.resourceType?.())) &&
+    matches(request.url(), options.urlPattern ?? options.url);
+  const track = (promise) => {
+    pending.add(promise);
+    promise.finally(() => pending.delete(promise)).catch(() => {});
+  };
+  const request = (request) => {
+    if (!allowed(request)) {
+      return;
+    }
+    ids.set(request, ++sequence);
+    track(
+      record('network.request', {
+        ...details(request),
+        headers: redactHeaders(request.headers?.()),
+        ...(options.bodies &&
+        ['document', 'xhr', 'fetch'].includes(request.resourceType?.()) &&
+        request.postData?.()
+          ? {
+              postData: Buffer.from(
+                redactText(request.postData(), privacy, {
+                  kind: 'network',
+                  name: 'postData',
+                })
+              )
+                .subarray(0, maxBodyBytes)
+                .toString('utf8'),
+              postDataTruncated:
+                Buffer.byteLength(request.postData()) > maxBodyBytes,
+            }
+          : {}),
+      })
+    );
+  };
+  const response = (response) => {
+    const request = response.request();
+    if (!ids.has(request)) {
+      return;
+    }
+    const saturated = pending.size >= 32;
+    track(
+      (async () => {
+        const headers = redactHeaders(
+          await withDeadline(
+            Promise.resolve(response.allHeaders?.() ?? response.headers()),
+            5000,
+            'network headers'
+          )
+        );
+        const payload = {
+          ...details(request),
+          status: response.status(),
+          statusText: response.statusText?.() ?? '',
+          headers,
+          contentType: headers['content-type'] ?? null,
+        };
+        const resourceType = payload.resourceType;
+        if (
+          !saturated &&
+          options.bodies &&
+          ['document', 'xhr', 'fetch'].includes(resourceType)
+        ) {
+          let length = Number(headers['content-length']);
+          if (!Number.isFinite(length) && request.sizes) {
+            const sizes = await withDeadline(
+              request.sizes(),
+              5000,
+              'network sizes'
+            ).catch(() => ({}));
+            length = sizes.responseBodySize ?? NaN;
+          }
+          if (!isTextBody(payload.contentType ?? '')) {
+            payload.body = {
+              size: Number.isFinite(length) ? length : null,
+              omitted: 'binary',
+            };
+          } else if (Number.isFinite(length) && length > maxBodyBytes) {
+            payload.body = {
+              size: length,
+              truncated: true,
+              omitted: 'size limit',
+            };
+          } else {
+            try {
+              const data = Buffer.from(
+                await withDeadline(
+                  response.body?.() ?? response.buffer(),
+                  5000,
+                  'network body'
+                )
+              );
+              payload.body = {
+                size: data.length,
+                truncated: data.length > maxBodyBytes,
+                encoding: 'utf8',
+                data: Buffer.from(
+                  redactText(data.toString('utf8'), privacy, {
+                    kind: 'network',
+                    name: 'responseBody',
+                  })
+                )
+                  .subarray(0, maxBodyBytes)
+                  .toString('utf8'),
+              };
+            } catch (error) {
+              payload.bodyError = error.message;
+            }
+          }
+        }
+        if (saturated && options.bodies) {
+          payload.bodyError = 'body capture concurrency limit';
+        }
+        await record('network.response', payload);
+      })().catch((error) => note(`network capture: ${error.message}`))
+    );
+  };
+  page.on('request', request);
+  page.on('response', response);
+  return async () => {
+    page.off('request', request);
+    page.off('response', response);
+    await Promise.allSettled([...pending]);
+  };
+}
+
+export async function writeHar(opened, output) {
+  await fs.writeFile(
+    output ?? path.join(opened.path, 'network.har'),
+    JSON.stringify(networkHar(opened.events), null, 2),
+    { mode: 0o600 }
+  );
+}