browser-commander 0.28.0 → 0.30.0

package/src/traces/commander-actions.js

@@ -0,0 +1,146 @@
+/** One wrapper per commander, even when explicit and automatic traces overlap. */
+import { withDeadline } from './deadline.js';
+import { framesOf } from './init-script.js';
+
+const commanders = new WeakMap();
+
+let actionSequence = 0;
+
+function labelActionInPage(argument, elementOptions) {
+  const { marker, remove } = elementOptions ?? argument;
+  const actions = (window.__browserCommanderActions__ ??= []);
+  if (remove) {
+    const index = actions.findIndex((action) => action.id === marker.id);
+    if (index >= 0) {
+      actions.splice(index, 1);
+    }
+  } else {
+    const focused =
+      !marker.selector && marker.types.includes('keydown')
+        ? document.activeElement
+        : null;
+    actions.push({
+      ...marker,
+      element: elementOptions
+        ? argument
+        : focused !== document.body && focused !== document.documentElement
+          ? focused
+          : null,
+    });
+  }
+}
+
+export function observeCommanderActions({
+  commander,
+  page,
+  names,
+  targetOf,
+  record,
+  identity,
+  now,
+  note,
+}) {
+  if (!commander) {
+    return () => {};
+  }
+  let state = commanders.get(commander);
+  if (!state) {
+    state = { observers: new Set(), originals: new Map() };
+    commanders.set(commander, state);
+    for (const name of names) {
+      const original = commander[name];
+      if (typeof original !== 'function') {
+        continue;
+      }
+      const wrapper = async (...args) => {
+        await commander.ready;
+        const observing = [...state.observers].map((observer) => ({
+          observer,
+          started: observer.now(),
+          actionId: observer.identity?.nextActionId() ?? null,
+        }));
+        const marker = {
+          id: ++actionSequence,
+          selector: targetOf(args[0]),
+          types: /^(click|check)/.test(name)
+            ? ['pointerdown', 'click', 'submit', 'contextmenu']
+            : /^(fill|typeText|pressKey)/.test(name)
+              ? ['keydown', 'submit', 'click']
+              : [],
+        };
+        const element = args[0]?.locatorOrElement ?? args[0]?.selector;
+        const labelAction = async (remove) => {
+          try {
+            if (marker.types.length) {
+              const config = { marker, remove };
+              const frames =
+                remove || (!marker.selector && marker.types.includes('keydown'))
+                  ? framesOf(page)
+                  : [];
+              const target = element?.first?.() ?? element;
+              const work = frames.length
+                ? Promise.all(
+                    frames.map((frame) =>
+                      frame.evaluate(labelActionInPage, config)
+                    )
+                  )
+                : !remove && typeof target?.evaluate === 'function'
+                  ? target.evaluate(labelActionInPage, config, {
+                      timeout: 1000,
+                    })
+                  : page.evaluate?.(labelActionInPage, config);
+              await withDeadline(
+                Promise.resolve(work),
+                1000,
+                'commander input label'
+              );
+            }
+          } catch (error) {
+            note?.(`could not label commander input: ${error.message}`);
+          }
+        };
+        await labelAction(false);
+        let error;
+        try {
+          return await original.apply(commander, args);
+        } catch (failure) {
+          error = failure;
+          throw failure;
+        } finally {
+          await labelAction(true);
+          await Promise.all(
+            observing.map(({ observer, started, actionId }) =>
+              observer.record('interaction', {
+                actionId,
+                source: 'commander',
+                action: name,
+                target: targetOf(args[0]),
+                durationMs: observer.now() - started,
+                ok: !error,
+                ...(error ? { error: error.message } : {}),
+              })
+            )
+          );
+        }
+      };
+      state.originals.set(name, { original, wrapper });
+      commander[name] = wrapper;
+    }
+  }
+  const observer = { record, identity, now };
+  if (record) {
+    state.observers.add(observer);
+  }
+  return () => {
+    state.observers.delete(observer);
+    if (state.observers.size) {
+      return;
+    }
+    for (const [name, { original, wrapper }] of state.originals) {
+      if (commander[name] === wrapper) {
+        commander[name] = original;
+      }
+    }
+    commanders.delete(commander);
+  };
+}

package/src/traces/debug-session.js

@@ -0,0 +1,119 @@
+import path from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { startTrace } from './recorder.js';
+
+const sessions = new WeakMap();
+const disabledPages = new WeakSet();
+
+export function debugEnabled(options = {}) {
+  return (
+    options.debug !== false &&
+    process.env.BROWSER_COMMANDER_DEBUG !== '0' &&
+    !(options.debug === undefined && disabledPages.has(options.page))
+  );
+}
+
+/** A launched page and its commander share one recording and one output path. */
+export async function ensureDebugSession(options) {
+  const { page, commander } = options;
+  const existing = sessions.get(page);
+  if (!debugEnabled(options)) {
+    disabledPages.add(page);
+    await existing?.stop();
+    return null;
+  }
+  disabledPages.delete(page);
+  let session = existing;
+  if (!session) {
+    const config =
+      options.debug && typeof options.debug === 'object' ? options.debug : {};
+    const output = path.resolve(
+      config.output ??
+        options.output ??
+        process.env.BROWSER_COMMANDER_TRACE ??
+        path.join(
+          'browser-commander-traces',
+          `${new Date().toISOString().replaceAll(':', '-')}-${process.pid}-${randomUUID().slice(0, 8)}`
+        )
+    );
+    session = { path: output };
+    sessions.set(page, session);
+    session.ready = startTrace({
+      ...config,
+      output,
+      page,
+      commander: commander ?? {
+        page,
+        engine: options.engine,
+        downloads: options.downloads,
+      },
+      checkpointOnNavigation: config.checkpointOnNavigation ?? true,
+    })
+      .then((trace) => {
+        console.error(`[browser-commander] Recording session to ${output}`);
+        return trace;
+      })
+      .catch((error) => {
+        sessions.delete(page);
+        console.warn(
+          `[browser-commander] Could not start recording: ${error.message}`
+        );
+        return null;
+      });
+    let stopping;
+    session.stop = () =>
+      (stopping ??= session.ready.then(async (trace) => {
+        try {
+          return await trace?.stop();
+        } finally {
+          if (sessions.get(page) === session) {
+            sessions.delete(page);
+          }
+        }
+      }));
+  }
+  const trace = await session.ready;
+  if (!trace) {
+    return null;
+  }
+  if (commander) {
+    trace.attachCommander(commander);
+  }
+  return session;
+}
+
+/** Finish trace files before closing/detaching the engine that supplies them. */
+export async function attachLaunchDebug(result, options) {
+  const session = await ensureDebugSession({
+    ...options,
+    engine:
+      options.engine ??
+      (result.page.isWebDriverPage ? 'selenium' : 'playwright'),
+    page: result.page,
+    downloads: result.downloads,
+  });
+  result.tracePath = session?.path ?? null;
+  if (!session) {
+    return result;
+  }
+  const wrap = (owner, name) => {
+    const method = owner?.[name];
+    if (typeof method !== 'function') {
+      return;
+    }
+    owner[name] = async (...args) => {
+      let result;
+      try {
+        await session.stop();
+      } finally {
+        result = await method.apply(owner, args);
+      }
+      return result;
+    };
+  };
+  wrap(result, 'close');
+  wrap(result, 'detach');
+  wrap(result.browser, 'close');
+  wrap(result.browser, 'disconnect');
+  return result;
+}

package/src/traces/drain-timer.js

@@ -0,0 +1,25 @@
+/** Serialize background drains and wait for the active drain during teardown. */
+export function startTraceDrain({
+  mutations,
+  userActions,
+  captureState,
+  note,
+}) {
+  let task;
+  const timer = setInterval(() => {
+    if (!task) {
+      task = Promise.resolve()
+        .then(() => userActions.drain())
+        .then(() => mutations.drain(captureState.index))
+        .catch((error) => note(error.message))
+        .finally(() => {
+          task = null;
+        });
+    }
+  }, 500);
+  timer.unref?.();
+  return async () => {
+    clearInterval(timer);
+    await task;
+  };
+}

package/src/traces/links.js

@@ -583,7 +583,9 @@ export async function openTraceLinks(options = {}) {
   const file = await resolveOutput(output);
   const sections = chosenSections(include);
   const problems = [];
-  const seenDom = new Set();
+  // Only the current interval is remembered; completed intervals never recur.
+  let mutationMember = null;
+  let mutationOffset = 0;
 
   await fs.mkdir(path.dirname(file), { recursive: true });
   const handle = await fs.open(file, 'w', 0o600);
@@ -626,7 +628,7 @@ export async function openTraceLinks(options = {}) {
      * @param {Object} event - The event as written to the bundle
      * @returns {Promise<void>}
      */
-    async event(event) {
+    async event(event, source = {}) {
       if (!event) {
         return;
       }
@@ -636,7 +638,45 @@ export async function openTraceLinks(options = {}) {
         bundlePath &&
         [TRACE_EVENT.CHECKPOINT, TRACE_EVENT.MUTATIONS].includes(event.kind)
       ) {
-        const opened = await readTrace(bundlePath);
+        const root = source.root ?? bundlePath;
+        let batches = [];
+        if (event.kind === TRACE_EVENT.MUTATIONS) {
+          const member = path.join(root, event.member);
+          if (member !== mutationMember) {
+            mutationMember = member;
+            mutationOffset = 0;
+          }
+          const input = await fs.open(member, 'r');
+          try {
+            const size = (await input.stat()).size;
+            const bytes = Buffer.alloc(Math.max(0, size - mutationOffset));
+            await input.read(bytes, 0, bytes.length, mutationOffset);
+            mutationOffset = size;
+            batches = bytes
+              .toString('utf8')
+              .split('\n')
+              .filter(Boolean)
+              .map(JSON.parse);
+          } finally {
+            await input.close();
+          }
+        }
+        const opened = {
+          html: () =>
+            event.members?.html
+              ? fs.readFile(path.join(root, event.members.html), 'utf8')
+              : null,
+          state: async () =>
+            event.members?.state
+              ? JSON.parse(
+                  await fs.readFile(
+                    path.join(root, event.members.state),
+                    'utf8'
+                  )
+                )
+              : null,
+          mutations: () => batches,
+        };
         await append(
           await domLinks(
             {
@@ -644,8 +684,7 @@ export async function openTraceLinks(options = {}) {
               checkpoints: event.kind === TRACE_EVENT.CHECKPOINT ? [event] : [],
               events: event.kind === TRACE_EVENT.MUTATIONS ? [event] : [],
             },
-            dom,
-            seenDom
+            dom
           )
         );
       }

package/src/traces/mutation-stream.js

@@ -20,7 +20,7 @@ import {
   RECORDER_GLOBAL,
   stopMutationRecorderInPage,
 } from './page-capture.js';
-import { REDACTED } from './redaction.js';
+import { redactValue, REDACTED } from './redaction.js';
 import { TRACE_DROP_REASON, TRACE_EVENT } from './schema.js';
 
 /** In-page records kept before the recorder starts counting drops instead. */
@@ -57,11 +57,14 @@ export function createMutationStream(options) {
   } = options;
 
   const enabled = Boolean(domOptions.mutations);
+  const globalName = `${RECORDER_GLOBAL}${identity.traceId}`;
 
   /** What the in-page recorder is built with, wherever it is installed. */
   const recorderOptions = {
-    globalName: RECORDER_GLOBAL,
+    globalName,
     redactSelectors: privacyOptions.redactSelectors,
+    redactAttributes: privacyOptions.redactAttributes,
+    useDefaults: privacyOptions.useDefaults,
     redacted: REDACTED,
     ignoreSelectors: domOptions.ignoreSelectors ?? [],
     maxQueued: limits.maxQueuedMutations ?? DEFAULT_MAX_QUEUED_MUTATIONS,
@@ -166,7 +169,7 @@ export function createMutationStream(options) {
     }
     try {
       const drained = await withDeadline(
-        evaluateInFrames(drainMutationsInPage, RECORDER_GLOBAL),
+        evaluateInFrames(drainMutationsInPage, globalName),
         captureTimeoutMs,
         'trace mutation drain'
       );
@@ -195,13 +198,17 @@ export function createMutationStream(options) {
       // frame; one ordered file is what a replay reads.
       batches.sort((left, right) => (left.at ?? 0) - (right.at ?? 0));
 
-      const member = await bundle.writeMutations(index, batches);
+      const member = await bundle.writeMutations(
+        index,
+        redactValue(batches, privacyOptions)
+      );
       if (member) {
         await record(TRACE_EVENT.MUTATIONS, {
           member,
           batches: batches.length,
           checkpoint: index,
           frames: drained.length,
+          truncated: bundle.mutationTruncated,
         });
       }
       return member;
@@ -223,7 +230,7 @@ export function createMutationStream(options) {
       return;
     }
     try {
-      await evaluateInFrames(stopMutationRecorderInPage, RECORDER_GLOBAL);
+      await evaluateInFrames(stopMutationRecorderInPage, globalName);
     } catch (error) {
       note?.(`could not stop the in-page recorder: ${error.message}`);
     }

package/src/traces/network.js

@@ -1,6 +1,11 @@
 import fs from 'node:fs/promises';
 import path from 'node:path';
 import { withDeadline } from './deadline.js';
+import {
+  normalizePrivacyOptions,
+  redactText,
+  redactHeaders as privateHeaders,
+} from './redaction.js';
 
 const PRIVATE_HEADERS = new Set([
   'cookie',
@@ -9,11 +14,20 @@ const PRIVATE_HEADERS = new Set([
   'set-cookie',
 ]);
 export function redactHeaders(headers = {}) {
-  return Object.fromEntries(
-    Object.entries(headers).map(([key, value]) => [
-      key.toLowerCase(),
-      PRIVATE_HEADERS.has(key.toLowerCase()) ? '[redacted]' : value,
-    ])
+  return privateHeaders(
+    Object.fromEntries(
+      Object.entries(headers).map(([key, value]) => [
+        key.toLowerCase(),
+        PRIVATE_HEADERS.has(key.toLowerCase()) ? '[redacted]' : value,
+      ])
+    ),
+    normalizePrivacyOptions()
+  );
+}
+
+export function isTextBody(contentType = '') {
+  return /^(?:text\/|application\/(?:[\w.+-]*json|[\w.+-]*xml|javascript|x-www-form-urlencoded))|multipart\/form-data/i.test(
+    contentType
   );
 }
 
@@ -81,10 +95,12 @@ export function networkHar(events) {
           content: {
             size: response.body?.size ?? 0,
             mimeType: response.contentType ?? '',
-            ...(response.body
+            ...(response.body?.data !== undefined
               ? {
                   text: response.body.data,
-                  encoding: 'base64',
+                  ...(response.body.encoding === 'utf8'
+                    ? {}
+                    : { encoding: 'base64' }),
                   _truncated: response.body.truncated,
                 }
               : {}),
@@ -131,7 +147,13 @@ export function validateNetworkOptions(network) {
     );
   }
 }
-export function attachNetwork({ page, network, record, note }) {
+export function attachNetwork({
+  page,
+  network,
+  record,
+  note,
+  privacy = normalizePrivacyOptions(),
+}) {
   validateNetworkOptions(network);
   if (!network) {
     return async () => {};
@@ -172,7 +194,12 @@ export function attachNetwork({ page, network, record, note }) {
         ['document', 'xhr', 'fetch'].includes(request.resourceType?.()) &&
         request.postData?.()
           ? {
-              postData: Buffer.from(request.postData())
+              postData: Buffer.from(
+                redactText(request.postData(), privacy, {
+                  kind: 'network',
+                  name: 'postData',
+                })
+              )
                 .subarray(0, maxBodyBytes)
                 .toString('utf8'),
               postDataTruncated:
@@ -210,9 +237,26 @@ export function attachNetwork({ page, network, record, note }) {
           options.bodies &&
           ['document', 'xhr', 'fetch'].includes(resourceType)
         ) {
-          const length = Number(headers['content-length']);
-          if (Number.isFinite(length) && length > maxBodyBytes) {
-            payload.body = { size: length, truncated: true, data: '' };
+          let length = Number(headers['content-length']);
+          if (!Number.isFinite(length) && request.sizes) {
+            const sizes = await withDeadline(
+              request.sizes(),
+              5000,
+              'network sizes'
+            ).catch(() => ({}));
+            length = sizes.responseBodySize ?? NaN;
+          }
+          if (!isTextBody(payload.contentType ?? '')) {
+            payload.body = {
+              size: Number.isFinite(length) ? length : null,
+              omitted: 'binary',
+            };
+          } else if (Number.isFinite(length) && length > maxBodyBytes) {
+            payload.body = {
+              size: length,
+              truncated: true,
+              omitted: 'size limit',
+            };
           } else {
             try {
               const data = Buffer.from(
@@ -225,7 +269,15 @@ export function attachNetwork({ page, network, record, note }) {
               payload.body = {
                 size: data.length,
                 truncated: data.length > maxBodyBytes,
-                data: data.subarray(0, maxBodyBytes).toString('base64'),
+                encoding: 'utf8',
+                data: Buffer.from(
+                  redactText(data.toString('utf8'), privacy, {
+                    kind: 'network',
+                    name: 'responseBody',
+                  })
+                )
+                  .subarray(0, maxBodyBytes)
+                  .toString('utf8'),
               };
             } catch (error) {
               payload.bodyError = error.message;

package/src/traces/observers.js

@@ -8,11 +8,49 @@
  */
 
 import { TRACE_EVENT } from './schema.js';
+import { observeCommanderActions } from './commander-actions.js';
+
+/** Keep page observations when a launched trace is adopted by a commander. */
+export function attachRecorderObservers(options) {
+  const ownerSources = ['interaction', 'dialog', 'download', 'navigation'];
+  const detachers = attachTimelineObservers({
+    ...options,
+    commander: null,
+    eventSources: options.eventSources.filter(
+      (source) => !ownerSources.includes(source)
+    ),
+  });
+  let commanderDetachers = [],
+    attachedCommander,
+    hasCommander = false;
+  const attachCommander = (commander) => {
+    if (hasCommander && commander === attachedCommander) {
+      return;
+    }
+    hasCommander = true;
+    attachedCommander = commander;
+    for (const detach of commanderDetachers) {
+      detach();
+    }
+    commanderDetachers = attachTimelineObservers({
+      ...options,
+      commander,
+      eventSources: options.eventSources.filter((source) =>
+        ownerSources.includes(source)
+      ),
+    });
+  };
+  attachCommander(options.commander);
+  detachers.push(() => commanderDetachers.forEach((detach) => detach()));
+  return { detachers, attachCommander };
+}
 
 /** Commander methods recorded as interactions while a trace is running. */
 export const TRACED_INTERACTIONS = Object.freeze([
   'click',
   'clickButton',
+  'clickElement',
+  'check',
   'fill',
   'fillTextArea',
   'goto',
@@ -231,51 +269,17 @@ export function attachTimelineObservers(options) {
     };
   });
 
-  subscribe('interaction', () => {
-    if (!commander) {
-      return null;
-    }
-    const originals = new Map();
-    for (const name of TRACED_INTERACTIONS) {
-      const original = commander[name];
-      if (typeof original !== 'function') {
-        continue;
-      }
-      originals.set(name, original);
-      commander[name] = async (...args) => {
-        const startedMs = now();
-        const target = interactionTarget(args[0]);
-        // Named before the call, so everything the action causes can be traced
-        // back to it even when the action itself ends in an exception.
-        const actionId = identity?.nextActionId() ?? null;
-        try {
-          const result = await original(...args);
-          await record(TRACE_EVENT.INTERACTION, {
-            actionId,
-            action: name,
-            target,
-            durationMs: now() - startedMs,
-            ok: true,
-          });
-          return result;
-        } catch (error) {
-          await record(TRACE_EVENT.INTERACTION, {
-            actionId,
-            action: name,
-            target,
-            durationMs: now() - startedMs,
-            ok: false,
-            error: error.message,
-          });
-          throw error;
-        }
-      };
-    }
-    return () => {
-      for (const [name, original] of originals) {
-        commander[name] = original;
-      }
-    };
-  });
+  subscribe('interaction', () =>
+    observeCommanderActions({
+      commander,
+      page,
+      names: TRACED_INTERACTIONS,
+      targetOf: interactionTarget,
+      record,
+      identity,
+      now,
+      note,
+    })
+  );
   return detachers;
 }