browser-commander 0.26.3 → 0.28.0

package/src/traces/render.js

@@ -0,0 +1,168 @@
+import { writeCapture } from '../capture/files.js';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { readTrace } from './reader.js';
+import {
+  encodeAnimation,
+  decodePng,
+  encodePng,
+  resizeFrame,
+} from '../capture/encoding.js';
+import { encodeVideo } from '../capture/video.js';
+
+export function withinTime(record, options = {}) {
+  const time = (value) =>
+    typeof value === 'number' || /^\d+(\.\d+)?$/.test(value)
+      ? { offset: Number(value) * 1000 }
+      : {
+          date: Date.parse(
+            /^\d{2}:\d{2}(?::\d{2})?$/.test(value)
+              ? `${options.startedAt?.slice(0, 10)}T${value.length === 5 ? `${value}:00` : value}Z`
+              : value
+          ),
+        };
+  for (const [name, direction] of [
+    ['from', -1],
+    ['to', 1],
+  ]) {
+    if (options[name] === undefined) {
+      continue;
+    }
+    const bound = time(options[name]);
+    if (bound.date !== undefined && !Number.isFinite(bound.date)) {
+      throw new RangeError(`invalid ${name} timestamp`);
+    }
+    const value =
+      bound.offset !== undefined
+        ? (record.monotonicMs ?? 0) - (options.monotonicOrigin ?? 0)
+        : Date.parse(record.at);
+    if (
+      direction === -1
+        ? value < (bound.offset ?? bound.date)
+        : value > (bound.offset ?? bound.date)
+    ) {
+      return false;
+    }
+  }
+  return true;
+}
+export async function summarizeTrace(root, options = {}) {
+  const opened = await readTrace(root);
+  options = timelineOptions(opened, options);
+  const records = [...opened.events];
+  const seenMutations = new Set();
+  for (const event of opened.events.filter(
+    (event) => event.kind === 'mutations'
+  )) {
+    const key = `${event.segment ?? ''}:${event.checkpoint}`;
+    if (seenMutations.has(key)) {
+      continue;
+    }
+    seenMutations.add(key);
+    for (const batch of await opened.mutations(event.checkpoint)) {
+      for (const mutation of batch.records ?? []) {
+        records.push({
+          ...mutation,
+          kind: `dom.${mutation.kind}`,
+          at: new Date(batch.at).toISOString(),
+          monotonicMs: event.monotonicMs,
+        });
+      }
+    }
+  }
+  return records
+    .filter(
+      (record) =>
+        withinTime(record, options) &&
+        (!options.grep || JSON.stringify(record).includes(options.grep))
+    )
+    .sort(
+      (a, b) =>
+        Date.parse(a.at) - Date.parse(b.at) ||
+        (a.sequence ?? 0) - (b.sequence ?? 0)
+    );
+}
+export async function renderTrace(root, options = {}) {
+  const opened = await readTrace(root);
+  options = timelineOptions(opened, options);
+  const frames = [];
+  let compressedBytes = 0;
+  for (const event of opened.events.filter(
+    (event) => event.kind === 'checkpoint' && withinTime(event, options)
+  )) {
+    const member = event.members?.screenshot;
+    if (member) {
+      if (frames.length >= 1000) {
+        throw new RangeError('trace render exceeds 1000 frames');
+      }
+      const file = path.join(opened.path, member);
+      compressedBytes += (await fs.stat(file)).size;
+      if (compressedBytes > 64 * 1024 * 1024) {
+        throw new RangeError('trace render frames exceed 64 MiB');
+      }
+      frames.push(await fs.readFile(file));
+    }
+  }
+  if (frames.length === 0) {
+    throw new Error('trace has no checkpoint screenshots in this interval');
+  }
+  if (frames.length > 1000) {
+    throw new RangeError('trace render exceeds 1000 frames');
+  }
+  const format =
+    options.format ?? (path.extname(options.path ?? '').slice(1) || 'gif');
+  if (format === 'sheet') {
+    let total = 0;
+    const images = frames.map((frame) => {
+      if (frame.length < 33) {
+        throw new RangeError('frame must be a PNG');
+      }
+      total +=
+        frame.readUInt32BE(16) *
+        frame.readUInt32BE(20) *
+        4 *
+        Math.max(1, (options.scale ?? 1) ** 2);
+      if (total > 256 * 1024 * 1024) {
+        throw new RangeError('contact sheet frames exceed 256 MiB');
+      }
+      return resizeFrame(decodePng(frame), options.scale ?? 1);
+    });
+    const columns = options.columns ?? Math.min(4, images.length);
+    if (!Number.isInteger(columns) || columns < 1 || columns > 1000) {
+      throw new RangeError('columns must be 1–1000');
+    }
+    const width = Math.max(...images.map((image) => image.width)) * columns;
+    const rowHeight = Math.max(...images.map((image) => image.height));
+    const height = rowHeight * Math.ceil(images.length / columns);
+    if (width * height > 16_777_216) {
+      throw new RangeError('contact sheet exceeds 16 megapixels');
+    }
+    const data = new Uint8Array(width * height * 4);
+    images.forEach((image, index) => {
+      const x = (index % columns) * (width / columns),
+        y = Math.floor(index / columns) * rowHeight;
+      for (let row = 0; row < image.height; row++) {
+        data.set(
+          image.data.subarray(
+            row * image.width * 4,
+            (row + 1) * image.width * 4
+          ),
+          ((y + row) * width + x) * 4
+        );
+      }
+    });
+    const bytes = encodePng({ width, height, data });
+    return writeCapture(bytes, options.path);
+  }
+  return ['mp4', 'webm', 'mov'].includes(format)
+    ? encodeVideo(frames, { ...options, format })
+    : encodeAnimation(frames, { ...options, format });
+}
+
+function timelineOptions(opened, options) {
+  return {
+    ...options,
+    monotonicOrigin: opened.events[0]?.monotonicMs ?? 0,
+    startedAt: opened.events[0]?.at,
+  };
+}

package/src/traces/rolling.js

@@ -0,0 +1,180 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+/** Bounded complete bundles, each independently readable after a crash. */
+export async function startRollingTrace(options, start) {
+  const config = options.limits.rotate === true ? {} : options.limits.rotate;
+  const maxBytes =
+    config.maxBytes ?? options.limits.maxBundleBytes ?? 32 * 1024 * 1024;
+  const maxSegments = config.maxSegments ?? 4;
+  if (
+    !Number.isSafeInteger(maxBytes) ||
+    maxBytes < 1024 ||
+    !Number.isSafeInteger(maxSegments) ||
+    maxSegments < 1 ||
+    maxSegments > 100
+  ) {
+    throw new RangeError('invalid trace rotation bounds');
+  }
+  const root = path.resolve(options.output);
+  await fs.mkdir(root, { recursive: true, mode: 0o700 });
+  let current,
+    sequence = 0,
+    stopping,
+    failure,
+    ticking = false;
+  const segments = [];
+  let queue = Promise.resolve();
+  const enqueue = (action) => {
+    const task = queue.then(action);
+    queue = task.catch((error) => {
+      failure ??= error;
+    });
+    return task;
+  };
+  async function open() {
+    const name = `segment-${String(++sequence).padStart(6, '0')}`;
+    const output = path.join(root, name);
+    current = await start({
+      ...options,
+      output,
+      limits: { ...options.limits, rotate: false, maxBundleBytes: maxBytes },
+      links: options.links
+        ? { ...options.links, output: path.join(output, 'trace.lino') }
+        : null,
+    });
+    segments.push(name);
+    while (segments.length > maxSegments) {
+      await fs.rm(path.join(root, segments.shift()), {
+        recursive: true,
+        force: true,
+      });
+    }
+    const temporary = path.join(root, 'segments.json.tmp');
+    await fs.writeFile(temporary, JSON.stringify({ segments }), {
+      mode: 0o600,
+    });
+    await fs.rename(temporary, path.join(root, 'segments.json'));
+  }
+  await open();
+  const timer = setInterval(() => {
+    if (!stopping && !ticking) {
+      ticking = true;
+      enqueue(async () => {
+        if (current.bytesWritten >= maxBytes * 0.8) {
+          await current.stop();
+          await open();
+        }
+      })
+        .catch(() => clearInterval(timer))
+        .finally(() => {
+          ticking = false;
+        });
+    }
+  }, 100);
+  timer.unref?.();
+  return {
+    path: root,
+    mode: options.mode,
+    checkpoint: (...args) =>
+      enqueue(() => {
+        if (stopping) {
+          throw new Error('trace has stopped');
+        }
+        return current.checkpoint(...args);
+      }),
+    event: (...args) =>
+      enqueue(async () => {
+        if (stopping) {
+          throw new Error('trace has stopped');
+        }
+        if (
+          current.bytesWritten +
+            Buffer.byteLength(JSON.stringify(args)) +
+            256 >=
+          maxBytes * 0.8
+        ) {
+          await current.stop();
+          await open();
+        }
+        return current.event(...args);
+      }),
+    stop: (...args) => {
+      stopping ??= (async () => {
+        clearInterval(timer);
+        await queue;
+        const result = await current.stop(...args);
+        if (args[0]?.discard) {
+          await fs.rm(root, { recursive: true, force: true });
+        }
+        if (failure) {
+          throw failure;
+        }
+        return { ...result, path: root, segments: [...segments] };
+      })();
+      return stopping;
+    },
+  };
+}
+
+/** Combine segment timelines while keeping checkpoint member paths distinct. */
+export async function readRollingTrace(root, index, read) {
+  const events = [],
+    checkpoints = [],
+    locations = new Map();
+  let manifest,
+    truncated = false,
+    number = 0;
+  for (const segment of index.segments) {
+    if (!/^segment-\d+$/.test(segment)) {
+      throw new Error('invalid trace segment path');
+    }
+    const opened = await read(path.join(root, segment));
+    manifest = opened.manifest;
+    truncated ||= opened.truncated;
+    const ids = new Map();
+    for (const checkpoint of opened.checkpoints) {
+      const id = ++number;
+      ids.set(checkpoint.index, id);
+      locations.set(id, { opened, index: checkpoint.index });
+      checkpoints.push({
+        ...checkpoint,
+        index: id,
+        members: Object.fromEntries(
+          Object.entries(checkpoint.members).map(([key, value]) => [
+            key,
+            `${segment}/${value}`,
+          ])
+        ),
+      });
+    }
+    for (const event of opened.events) {
+      const mapped = { ...event, segment };
+      if (event.kind === 'checkpoint') {
+        Object.assign(
+          mapped,
+          checkpoints.find(
+            (checkpoint) => checkpoint.index === ids.get(event.index)
+          )
+        );
+      }
+      if (event.kind === 'mutations') {
+        mapped.checkpoint = ids.get(event.checkpoint) ?? 0;
+      }
+      events.push(mapped);
+    }
+  }
+  return {
+    path: root,
+    manifest,
+    events,
+    checkpoints,
+    truncated,
+    html: (index) =>
+      locations.get(index)?.opened.html(locations.get(index).index),
+    state: (index) =>
+      locations.get(index)?.opened.state(locations.get(index).index),
+    mutations: (index) =>
+      locations.get(index)?.opened.mutations(locations.get(index).index) ?? [],
+  };
+}

package/src/traces/start-cleanup.js

@@ -0,0 +1,15 @@
+/** Release resources when startup fails before returning a running trace. */
+export async function abortTraceStart(resources) {
+  const { detachers, detachNetwork, video, mutations, linksSink, bundle } =
+    resources;
+  for (const detach of detachers.reverse()) {
+    await Promise.resolve()
+      .then(detach)
+      .catch(() => {});
+  }
+  await detachNetwork().catch(() => {});
+  await video?.stop().catch(() => {});
+  await mutations.stop().catch(() => {});
+  await linksSink?.discard().catch(() => {});
+  await bundle.abort();
+}

package/src/traces/stop.js

@@ -0,0 +1,159 @@
+import { createManifest, TRACE_EVENT, TRACE_OUTCOME } from './schema.js';
+import { readTrace } from './reader.js';
+import { writeHar } from './network.js';
+import { gzipTrace } from './storage.js';
+import { abortTraceStart } from './start-cleanup.js';
+export function createTraceStop(context) {
+  const {
+    getStopped,
+    setStopped,
+    options,
+    detachers,
+    navigationCapture,
+    video,
+    detachNetwork,
+  } = context;
+  const {
+    mutations,
+    captureState,
+    record,
+    note,
+    bundle,
+    checkpoints,
+    linksSink,
+    network,
+  } = context;
+  const {
+    recorderMode,
+    startedAt,
+    now,
+    commanderVersion,
+    engine,
+    domOptions,
+    eventSources,
+    privacyOptions,
+    limits,
+  } = context;
+  let stopping;
+  async function stop(stopOptions = {}) {
+    if (getStopped()) {
+      return getStopped();
+    }
+    const { discard = false, error = null } = stopOptions;
+
+    {
+      for (const detach of detachers.splice(0)) {
+        try {
+          await detach();
+        } catch (error) {
+          note(`could not detach a listener: ${error.message}`);
+        }
+      }
+      await navigationCapture();
+    }
+    if (video) {
+      const recording = await video.stop();
+      if (recording.bytes) {
+        await bundle.writeMember(
+          `recording.${recording.format}`,
+          recording.bytes
+        );
+      }
+    }
+    await detachNetwork();
+    await mutations.drain(captureState.index);
+    if (error) {
+      await record(TRACE_EVENT.PAGE_ERROR, {
+        message: error.message ?? String(error),
+        stack: error.stack ?? null,
+        fatal: true,
+      });
+    }
+    await record(TRACE_EVENT.TRACE_STOP, { discarded: discard });
+    setStopped(true);
+
+    // The documents that exist stop observing here; the engine's init-script
+    // registration is removed with the detachers below.
+    await mutations.stop();
+
+    for (const detach of detachers.reverse()) {
+      try {
+        await detach();
+      } catch (detachError) {
+        note(`could not detach a listener: ${detachError.message}`);
+      }
+    }
+
+    const manifest = await bundle.close(
+      createManifest({
+        mode: recorderMode,
+        startedAt,
+        stoppedAt: new Date(now()).toISOString(),
+        outcome: TRACE_OUTCOME.COMPLETE,
+        commanderVersion,
+        engine,
+        dom: domOptions,
+        events: eventSources,
+        replay: {
+          checkpoints: true,
+          mutations: Boolean(domOptions.mutations),
+          childListPositions: Boolean(domOptions.mutations),
+          liveState:
+            Boolean(domOptions.mutations) && domOptions.liveState !== false,
+          identifiers: true,
+        },
+        privacy: {
+          redactSelectors: privacyOptions.redactSelectors,
+          redactAttributes: privacyOptions.redactAttributes,
+          redactQueryParams: privacyOptions.redactQueryParams,
+          hasCallback: Boolean(privacyOptions.redact),
+        },
+        limits,
+      })
+    );
+
+    // Closed after the manifest, because the closing link reports the outcome
+    // the manifest settled on, and the control diffs are read back out of the
+    // finished bundle rather than kept in memory for the length of a run.
+    if (linksSink) {
+      await linksSink.close({ manifest, bundlePath: bundle.root });
+    }
+    if (network?.har) {
+      await writeHar(
+        await readTrace(bundle.root),
+        typeof network.har === 'string' ? network.har : undefined
+      );
+    }
+    if (options.gzip || limits.gzip) {
+      await gzipTrace(bundle.root);
+    }
+
+    const result = {
+      path: bundle.root,
+      manifest,
+      checkpoints: [...checkpoints],
+      problems: [...bundle.problems, ...(linksSink?.problems ?? [])],
+      links: linksSink?.path ?? null,
+    };
+    setStopped(result);
+
+    if (discard) {
+      const fs = await import('node:fs/promises');
+      await fs.rm(bundle.root, { recursive: true, force: true });
+      // An export of a bundle that no longer exists points at nothing.
+      await linksSink?.discard();
+      result.discarded = true;
+    }
+
+    return result;
+  }
+
+  return (options) => {
+    stopping ??= stop(options).catch(async (error) => {
+      setStopped(true);
+      await abortTraceStart({ ...context, video: null }).catch(() => {});
+      throw error;
+    });
+    return stopping;
+  };
+}

package/src/traces/storage.js

@@ -0,0 +1,43 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { gzip, gunzip } from 'node:zlib';
+import { promisify } from 'node:util';
+const compress = promisify(gzip),
+  decompress = promisify(gunzip);
+
+export async function readTraceText(file) {
+  try {
+    return await fs.readFile(file, 'utf8');
+  } catch (error) {
+    if (error.code !== 'ENOENT') {
+      throw error;
+    }
+    try {
+      return (await decompress(await fs.readFile(`${file}.gz`))).toString(
+        'utf8'
+      );
+    } catch (error) {
+      if (error.code === 'ENOENT') {
+        return null;
+      }
+      throw error;
+    }
+  }
+}
+
+/** Compress closed NDJSON members only; active traces remain crash-readable. */
+export async function gzipTrace(root) {
+  for (const item of await fs.readdir(root, { withFileTypes: true })) {
+    const file = path.join(root, item.name);
+    if (item.isDirectory()) {
+      await gzipTrace(file);
+    } else if (item.name.endsWith('.ndjson')) {
+      const temporary = `${file}.gz.tmp`;
+      await fs.writeFile(temporary, await compress(await fs.readFile(file)), {
+        mode: 0o600,
+      });
+      await fs.rename(temporary, `${file}.gz`);
+      await fs.rm(file);
+    }
+  }
+}

package/src/utilities/subprocess.js

@@ -1,5 +1,37 @@
 import { ProcessRunner } from 'command-stream/process-runner';
 
+/** Start an explicitly persistent process, with no parent-owned pipes or signal forwarding. */
+export async function startDetachedProcess(file, args = [], options = {}) {
+  const runner = runnerFor(file, args, { ...options, capture: false });
+  runner.start();
+  await new Promise((resolve, reject) => {
+    runner.child.once('spawn', resolve);
+    runner.child.once('error', reject);
+  });
+  const child = runner.child;
+  // command-stream 1.x starts noninteractive children in their own process
+  // group. Release its controller-owned cancellation registry while retaining
+  // the child's exit listeners. Keep this adapter isolated until it exposes a
+  // public detach operation; normal subprocesses retain managed cancellation.
+  runner._child = null;
+  runner._abortController = null;
+  runner.finished = true;
+  runner._cleanup();
+  Promise.resolve(runner).catch(() => {});
+  child.stderrTail = '';
+  child.stderr?.on('data', (chunk) => {
+    child.stderrTail = (child.stderrTail + String(chunk)).slice(-8192);
+  });
+  child.on('error', (error) => {
+    child.spawnError = error;
+  });
+  child.unref();
+  child.stdout?.unref?.();
+  child.stderr?.unref?.();
+  child.stdin?.unref?.();
+  return child;
+}
+
 /**
  * Every subprocess Browser Commander starts - browsers, WebDriver servers, the
  * engine bridge, credential tools (`security`, `secret-tool`, `kwallet-query`,
@@ -14,12 +46,16 @@ import { ProcessRunner } from 'command-stream/process-runner';
  * optional PTY and terminal-rendering modules are never loaded.
  */
 
-function runnerFor(file, args, { env, cwd, stdin = 'ignore', killGrace }) {
+function runnerFor(
+  file,
+  args,
+  { env, cwd, stdin = 'ignore', killGrace, capture = true }
+) {
   return new ProcessRunner(
     { mode: 'exec', file, args },
     {
       mirror: false,
-      capture: true,
+      capture,
       stdin,
       ...(env ? { env } : {}),
       ...(cwd ? { cwd } : {}),
@@ -102,7 +138,7 @@ export async function runCommand(file, args = [], options = {}) {
  */
 export function startProcess(file, args = [], options = {}) {
   const { env, cwd, forwardOutput = false, killGrace = 2000 } = options;
-  const runner = runnerFor(file, args, { env, cwd, killGrace });
+  const runner = runnerFor(file, args, { env, cwd, killGrace, capture: false });
   return new ManagedProcess(runner, { forwardOutput });
 }
 
@@ -136,6 +172,8 @@ export class ManagedProcess {
     this.runner = runner;
     this.exitCode = null;
     this.signalCode = null;
+    this.stderrTail = '';
+    this.spawnError = null;
     this.stdout = new OutputChannel();
     this.stderr = new OutputChannel();
     this.exitListeners = new Set();
@@ -146,6 +184,7 @@ export class ManagedProcess {
       this.stdout.emit(chunk);
     });
     runner.on('stderr', (chunk) => {
+      this.stderrTail = (this.stderrTail + String(chunk)).slice(-8192);
       if (forwardOutput) {
         process.stderr.write(chunk);
       }
@@ -153,9 +192,13 @@ export class ManagedProcess {
     });
     runner.start();
     this.exited = Promise.resolve(runner)
-      .catch((error) => ({ code: error?.code ?? 1 }))
+      .catch((error) => {
+        this.spawnError = error;
+        return { code: error?.code ?? 1 };
+      })
       .then((result) => {
         this.exitCode = typeof result.code === 'number' ? result.code : 1;
+        this.signalCode = result.signal ?? null;
         for (const listener of this.exitListeners) {
           listener(this.exitCode);
         }