browser-commander 0.17.2 → 0.18.0

package/src/tests/index.js

@@ -11,6 +11,25 @@ import path from 'node:path';
 import { test as baseTest } from 'test-anywhere';
 import { launchBrowser } from '../browser/launcher.js';
 import { makeBrowserCommander } from '../factory.js';
+import {
+  finishScenarioTrace,
+  resolveTestDownloads,
+  startScenarioTrace,
+} from './tracing.js';
+
+export {
+  TEST_SCREENSHOT_MODES,
+  TEST_TRACE_MODES,
+  TRACE_BUNDLE_SUFFIX,
+  TEST_DOWNLOADS_DIRNAME,
+  finishScenarioTrace,
+  normalizeTestScreenshots,
+  normalizeTestTrace,
+  resolveTestDownloads,
+  resolveTraceSetting,
+  startScenarioTrace,
+  traceOutputPath,
+} from './tracing.js';
 
 export {
   test,
@@ -406,11 +425,14 @@ export async function createBrowserFixture(options = {}) {
     ...launchOptions,
     engine: normalizedEngine,
   });
-  const { browser, page } = launched;
+  const { browser, page, downloads = null } = launched;
 
   try {
     const commander = makeCommander({
       page,
+      // A managed download manager belongs to the commander that runs the
+      // test, so `commander.downloads` and the trace see the same files.
+      ...(downloads ? { downloads } : {}),
       ...commanderOptions,
     });
 
@@ -418,6 +440,7 @@ export async function createBrowserFixture(options = {}) {
       engine: normalizedEngine,
       browser,
       page,
+      downloads,
       commander,
       async close() {
         if (typeof commander.destroy === 'function') {
@@ -563,36 +586,67 @@ export function runBrowserScenario(scenario, options = {}) {
     launch,
     makeCommander,
     artifactsDir = DEFAULT_ARTIFACTS_DIR,
+    trace = 'off',
+    screenshots,
+    traceOptions = {},
     timeoutMs,
     attempt = 1,
   } = options;
   const testId = browserTestId(normalizedScenario, engine);
+  const scenarioArtifactsDir = normalizedScenario.artifactsDir ?? artifactsDir;
+  const safeName = sanitizeArtifactName(`${testId}-${engine}`);
+  const mergedLaunchOptions = mergeOptions(
+    launchOptions,
+    normalizedScenario.launchOptions
+  );
+  // A download a test made is an artifact of that test, so it is kept beside
+  // the trace that references it rather than in the user's Downloads folder.
+  const downloads = resolveTestDownloads(mergedLaunchOptions.downloads, {
+    artifactsDir: scenarioArtifactsDir,
+    safeName,
+  });
 
   return withBrowserCommander(
     {
       engine,
       launch,
       makeCommander,
-      launchOptions: mergeOptions(
-        launchOptions,
-        normalizedScenario.launchOptions
-      ),
+      launchOptions: downloads
+        ? { ...mergedLaunchOptions, downloads }
+        : mergedLaunchOptions,
       commanderOptions: mergeOptions(
         commanderOptions,
         normalizedScenario.commanderOptions
       ),
     },
     async (fixture) => {
+      const started = await startScenarioTrace({
+        commander: fixture.commander,
+        page: fixture.page,
+        trace: normalizedScenario.trace ?? trace,
+        screenshots: normalizedScenario.screenshots ?? screenshots,
+        artifactsDir: scenarioArtifactsDir,
+        safeName,
+        attempt,
+        traceOptions: mergeOptions(
+          traceOptions,
+          normalizedScenario.traceOptions
+        ),
+      });
+
       try {
-        return await runWithTimeout(
+        const result = await runWithTimeout(
           () =>
             normalizedScenario.fn({
               ...fixture,
+              trace: started?.trace ?? null,
               testInfo: {
                 id: testId,
                 name: normalizedScenario.name,
                 engine,
                 attempt,
+                artifactsDir: scenarioArtifactsDir,
+                tracePath: started?.path ?? null,
                 tags: normalizedScenario.tags || [],
               },
             }),
@@ -601,14 +655,24 @@ export function runBrowserScenario(scenario, options = {}) {
             testId,
           }
         );
+        await finishScenarioTrace(started, { log: fixture.commander?.log });
+        return result;
       } catch (error) {
-        await writeFailureArtifacts({
-          page: fixture.page,
+        const stopped = await finishScenarioTrace(started, {
           error,
-          testId,
-          engine,
-          artifactsDir: normalizedScenario.artifactsDir ?? artifactsDir,
+          log: fixture.commander?.log,
         });
+        // The trace bundle is the failure artifact when one was recorded; the
+        // standalone error file and screenshot stay for runs without a trace.
+        if (!stopped) {
+          await writeFailureArtifacts({
+            page: fixture.page,
+            error,
+            testId,
+            engine,
+            artifactsDir: scenarioArtifactsDir,
+          });
+        }
         throw error;
       }
     }

package/src/tests/tracing.js

@@ -0,0 +1,236 @@
+/**
+ * Trace and download artifacts for the Browser Commander test runner
+ * (issues #87 and #88).
+ *
+ * A failing browser test is only useful if it can be read afterwards. The
+ * runner therefore records the same portable trace bundle every other
+ * consumer gets, instead of inventing a second artifact format, and points
+ * managed downloads at the same artifact directory so a trace can reference
+ * files that are still on disk.
+ */
+
+import path from 'node:path';
+import { startTrace } from '../traces/recorder.js';
+import { writeTraceViewer } from '../traces/viewer.js';
+import { TRACE_MODE } from '../traces/schema.js';
+
+/** Trace settings a scenario may ask for, in Playwright's vocabulary. */
+export const TEST_TRACE_MODES = Object.freeze([
+  'off',
+  'on',
+  'retain-on-failure',
+  'on-first-retry',
+]);
+
+/** Screenshot settings a scenario may ask for. */
+export const TEST_SCREENSHOT_MODES = Object.freeze([
+  'off',
+  'on',
+  'only-on-failure',
+]);
+
+/** Suffix that marks a trace bundle directory. */
+export const TRACE_BUNDLE_SUFFIX = '.bc-trace';
+
+/** Directory managed downloads use inside a test's artifact directory. */
+export const TEST_DOWNLOADS_DIRNAME = 'downloads';
+
+/**
+ * Read a scenario's `trace` option.
+ *
+ * @param {boolean|string} [trace] - `true`, `false` or one of TEST_TRACE_MODES
+ * @returns {string} One of TEST_TRACE_MODES
+ */
+export function normalizeTestTrace(trace) {
+  if (trace === undefined || trace === null || trace === false) {
+    return 'off';
+  }
+  if (trace === true) {
+    return 'on';
+  }
+  if (!TEST_TRACE_MODES.includes(trace)) {
+    throw new Error(`trace must be one of ${TEST_TRACE_MODES.join(', ')}`);
+  }
+  return trace;
+}
+
+/**
+ * Read a scenario's `screenshots` option.
+ *
+ * @param {boolean|string} [screenshots] - `true`, `false` or one of
+ *   TEST_SCREENSHOT_MODES
+ * @returns {string} One of TEST_SCREENSHOT_MODES
+ */
+export function normalizeTestScreenshots(screenshots) {
+  if (screenshots === undefined || screenshots === null) {
+    return 'only-on-failure';
+  }
+  if (screenshots === true) {
+    return 'on';
+  }
+  if (screenshots === false) {
+    return 'off';
+  }
+  if (!TEST_SCREENSHOT_MODES.includes(screenshots)) {
+    throw new Error(
+      `screenshots must be one of ${TEST_SCREENSHOT_MODES.join(', ')}`
+    );
+  }
+  return screenshots;
+}
+
+/**
+ * Decide whether this attempt records, and in which recorder mode.
+ *
+ * @param {string} trace - A value from `normalizeTestTrace()`
+ * @param {number} attempt - 1 for the first run, 2 for the first retry
+ * @returns {{recorderMode: string, retainOnFailure: boolean}|null} The
+ *   recorder settings, or null when this attempt does not record
+ */
+export function resolveTraceSetting(trace, attempt = 1) {
+  if (trace === 'off') {
+    return null;
+  }
+  if (trace === 'on-first-retry' && attempt < 2) {
+    return null;
+  }
+  return {
+    recorderMode:
+      trace === 'on' ? TRACE_MODE.CONTINUOUS : TRACE_MODE.RETAIN_ON_FAILURE,
+    retainOnFailure: trace !== 'on',
+  };
+}
+
+function screenshotSettingForRecorder(screenshots) {
+  if (screenshots === 'on') {
+    return true;
+  }
+  if (screenshots === 'off') {
+    return false;
+  }
+  return 'only-on-failure';
+}
+
+/**
+ * Where one attempt's trace bundle lives.
+ *
+ * @param {Object} options - `{artifactsDir, safeName, attempt}`
+ * @returns {string} The bundle directory
+ */
+export function traceOutputPath(options = {}) {
+  const { artifactsDir, safeName, attempt = 1 } = options;
+  const suffix = attempt > 1 ? `.attempt-${attempt}` : '';
+  return path.resolve(
+    artifactsDir,
+    `${safeName}${suffix}${TRACE_BUNDLE_SUFFIX}`
+  );
+}
+
+/**
+ * Point a scenario's managed downloads at its own artifact directory, so a
+ * completed download is still there when the trace that references it is read
+ * (issue #88).
+ *
+ * @param {boolean|Object} downloads - The scenario's `downloads` option
+ * @param {Object} options - `{artifactsDir, safeName}`
+ * @returns {boolean|Object|undefined} The download options to launch with
+ */
+export function resolveTestDownloads(downloads, options = {}) {
+  const { artifactsDir, safeName } = options;
+  if (!downloads || !artifactsDir) {
+    return downloads;
+  }
+  const settings = downloads === true ? {} : downloads;
+  if (settings.directory) {
+    return settings;
+  }
+  return {
+    ...settings,
+    directory: path.resolve(artifactsDir, safeName, TEST_DOWNLOADS_DIRNAME),
+  };
+}
+
+/**
+ * Start a trace for one scenario attempt.
+ *
+ * @param {Object} options - `{commander, page, trace, screenshots,
+ *   artifactsDir, safeName, attempt, traceOptions}`
+ * @returns {Promise<Object|null>} `{trace, path, retainOnFailure}`, or null
+ *   when this attempt does not record
+ */
+export async function startScenarioTrace(options = {}) {
+  const {
+    commander,
+    page,
+    trace,
+    screenshots,
+    artifactsDir,
+    safeName,
+    attempt = 1,
+    traceOptions = {},
+  } = options;
+
+  const setting = resolveTraceSetting(normalizeTestTrace(trace), attempt);
+  if (!setting || !artifactsDir) {
+    return null;
+  }
+
+  const running = await startTrace({
+    commander,
+    page,
+    output: traceOutputPath({ artifactsDir, safeName, attempt }),
+    mode: setting.recorderMode,
+    screenshots: screenshotSettingForRecorder(
+      normalizeTestScreenshots(screenshots)
+    ),
+    // A test that fails halfway is exactly where ordered DOM mutations pay
+    // for themselves, so they are on whenever the runner records.
+    dom: { mutations: true },
+    ...traceOptions,
+  });
+
+  return {
+    trace: running,
+    path: running.path,
+    retainOnFailure: setting.retainOnFailure,
+  };
+}
+
+/**
+ * Stop a scenario's trace, keeping it only when it is worth keeping.
+ *
+ * @param {Object|null} started - What `startScenarioTrace()` returned
+ * @param {Object} [options] - `{error, log}`
+ * @returns {Promise<Object|null>} The stopped trace, or null
+ */
+export async function finishScenarioTrace(started, options = {}) {
+  if (!started) {
+    return null;
+  }
+
+  const { error = null, log } = options;
+  const { trace, retainOnFailure } = started;
+  const note = (message) => log?.debug?.(`[trace] ${message}`);
+
+  try {
+    await trace.checkpoint(error ? 'failure' : 'final', {
+      actor: 'runner',
+      reason: error ? 'failure' : 'final',
+    });
+  } catch (checkpointError) {
+    note(`could not capture the final checkpoint: ${checkpointError.message}`);
+  }
+
+  const discard = Boolean(retainOnFailure) && !error;
+  const stopped = await trace.stop({ error, discard });
+
+  if (!discard) {
+    try {
+      await writeTraceViewer(stopped.path);
+    } catch (viewerError) {
+      note(`could not write the viewer: ${viewerError.message}`);
+    }
+  }
+
+  return stopped;
+}

package/src/traces/bundle.js

@@ -0,0 +1,323 @@
+/**
+ * Writing a trace bundle to disk (issue #87).
+ *
+ * The bundle is a directory with the documented layout, which `zip -r` turns
+ * into the archive form without re-encoding anything, and which `ls` and
+ * `cat` can already read.
+ *
+ * Every write here is best-effort by default: a trace exists to explain a run,
+ * so failing to record something must leave a visible gap rather than break
+ * the automation that was being recorded.
+ */
+
+import { createHash } from 'node:crypto';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import {
+  DEFAULT_MAX_BUNDLE_BYTES,
+  DEFAULT_MAX_RESOURCE_BYTES,
+  TRACE_DROP_REASON,
+  TRACE_EVENT,
+  TRACE_FILES,
+  TRACE_OUTCOME,
+  sequenceName,
+} from './schema.js';
+
+/** Owner-only file mode: a trace may hold form values and page content. */
+export const TRACE_FILE_MODE = 0o600;
+
+/** Owner-only directory mode for the bundle root. */
+export const TRACE_DIRECTORY_MODE = 0o700;
+
+/**
+ * Open a bundle for writing.
+ *
+ * @param {Object} options - `{output, limits, strict, now}`
+ * @returns {Promise<Object>} The bundle writer
+ */
+export async function openTraceBundle(options = {}) {
+  const {
+    output,
+    strict = false,
+    limits = {},
+    now = () => Date.now(),
+  } = options;
+
+  if (typeof output !== 'string' || output === '') {
+    throw new Error('trace output must be a path');
+  }
+
+  const root = path.resolve(output);
+  const maxBundleBytes = limits.maxBundleBytes ?? DEFAULT_MAX_BUNDLE_BYTES;
+  const maxResourceBytes =
+    limits.maxResourceBytes ?? DEFAULT_MAX_RESOURCE_BYTES;
+  const maxEventBytes = limits.maxEventBytes ?? maxResourceBytes;
+
+  await fs.mkdir(root, { recursive: true, mode: TRACE_DIRECTORY_MODE });
+
+  const eventsPath = path.join(root, TRACE_FILES.EVENTS);
+  const eventsHandle = await fs.open(eventsPath, 'a', TRACE_FILE_MODE);
+
+  let written = 0;
+  let sequence = 0;
+  // Appends are serialized. Listeners report what happened without waiting
+  // for the previous record to reach the disk, and a timeline whose lines
+  // arrive in a different order than the events did is not a timeline.
+  let appendQueue = Promise.resolve();
+  let dropped = 0;
+  const counts = { checkpoints: 0, events: 0, mutationBatches: 0 };
+  const problems = [];
+
+  const fits = (bytes) =>
+    bytes <= maxResourceBytes && written + bytes <= maxBundleBytes;
+
+  // A record of something that was dropped is the one write that must not be
+  // dropped in turn, so it answers to the bundle budget alone.
+  const fitsEvent = (bytes, retry) =>
+    written + bytes <= maxBundleBytes && (!retry || bytes <= maxEventBytes);
+
+  /**
+   * Record that something could not be written.
+   *
+   * @param {Object} record - `{reason, detail, member}`
+   * @returns {Promise<void>}
+   */
+  async function drop(record) {
+    dropped += 1;
+    problems.push(record);
+    if (strict) {
+      throw new Error(
+        `trace ${record.member ?? 'record'} dropped: ${record.reason}${
+          record.detail ? ` (${record.detail})` : ''
+        }`
+      );
+    }
+    // The drop is itself an event, so a reader sees the gap in the timeline
+    // instead of silently reading a shorter story than the one that happened.
+    await appendEvent(
+      { kind: TRACE_EVENT.DROPPED, ...record },
+      { retry: false }
+    );
+  }
+
+  /**
+   * Append one event to the timeline.
+   *
+   * @param {Object} event - Event fields; `sequence` and `at` are added
+   * @param {Object} [options] - `{retry}`
+   * @returns {Promise<Object|null>} The event as written
+   */
+  async function appendEvent(event, { retry = true } = {}) {
+    const record = {
+      sequence: ++sequence,
+      at: new Date(now()).toISOString(),
+      monotonicMs: Math.round(performance.now()),
+      ...event,
+    };
+    const line = `${JSON.stringify(record)}\n`;
+    const bytes = Buffer.byteLength(line);
+
+    if (!fitsEvent(bytes, retry)) {
+      if (retry) {
+        await drop({
+          reason: TRACE_DROP_REASON.SIZE_LIMIT,
+          member: TRACE_FILES.EVENTS,
+          detail: `${bytes} bytes`,
+        });
+      }
+      return null;
+    }
+
+    const pending = appendQueue.then(() => eventsHandle.write(line));
+    appendQueue = pending.then(
+      () => {},
+      () => {}
+    );
+
+    try {
+      await pending;
+      written += bytes;
+      counts.events += 1;
+      return record;
+    } catch (error) {
+      if (retry) {
+        await drop({
+          reason: TRACE_DROP_REASON.WRITE_FAILED,
+          member: TRACE_FILES.EVENTS,
+          detail: error.message,
+        });
+      }
+      return null;
+    }
+  }
+
+  /**
+   * Write one member of the bundle.
+   *
+   * @param {string} member - Path relative to the bundle root
+   * @param {string|Buffer} contents - What to write
+   * @returns {Promise<Object|null>} `{member, bytes}` or null when dropped
+   */
+  async function writeMember(member, contents) {
+    const data = Buffer.isBuffer(contents) ? contents : Buffer.from(contents);
+
+    if (!fits(data.length)) {
+      await drop({
+        reason: TRACE_DROP_REASON.SIZE_LIMIT,
+        member,
+        detail: `${data.length} bytes`,
+      });
+      return null;
+    }
+
+    const target = path.join(root, member);
+    try {
+      await fs.mkdir(path.dirname(target), {
+        recursive: true,
+        mode: TRACE_DIRECTORY_MODE,
+      });
+      await fs.writeFile(target, data, { mode: TRACE_FILE_MODE });
+      written += data.length;
+      return { member, bytes: data.length };
+    } catch (error) {
+      await drop({
+        reason: TRACE_DROP_REASON.WRITE_FAILED,
+        member,
+        detail: error.message,
+      });
+      return null;
+    }
+  }
+
+  /**
+   * Write a checkpoint's HTML, state and screenshot.
+   *
+   * @param {Object} checkpoint - `{index, html, state, screenshot}`
+   * @returns {Promise<Object>} What was written, by member name
+   */
+  async function writeCheckpoint({ index, html, state, screenshot }) {
+    const name = sequenceName(index);
+    const members = {};
+
+    if (typeof html === 'string') {
+      const result = await writeMember(
+        `${TRACE_FILES.CHECKPOINTS_DIR}/${name}.html`,
+        html
+      );
+      if (result) {
+        members.html = result.member;
+      }
+    }
+    if (state) {
+      const result = await writeMember(
+        `${TRACE_FILES.CHECKPOINTS_DIR}/${name}.state.json`,
+        `${JSON.stringify(state, null, 2)}\n`
+      );
+      if (result) {
+        members.state = result.member;
+      }
+    }
+    if (screenshot) {
+      const result = await writeMember(
+        `${TRACE_FILES.CHECKPOINTS_DIR}/${name}.png`,
+        screenshot
+      );
+      if (result) {
+        members.screenshot = result.member;
+      }
+    }
+
+    counts.checkpoints += 1;
+    return members;
+  }
+
+  /**
+   * Write one checkpoint's mutation batches.
+   *
+   * @param {number} index - Checkpoint the batches follow
+   * @param {Object[]} batches - Ordered mutation batches
+   * @returns {Promise<string|null>} Member name, or null when nothing was written
+   */
+  async function writeMutations(index, batches) {
+    if (!batches || batches.length === 0) {
+      return null;
+    }
+    const member = `${TRACE_FILES.MUTATIONS_DIR}/${sequenceName(index)}.ndjson`;
+    const body = `${batches.map((batch) => JSON.stringify(batch)).join('\n')}\n`;
+    const result = await writeMember(member, body);
+    if (result) {
+      counts.mutationBatches += batches.length;
+    }
+    return result ? result.member : null;
+  }
+
+  /**
+   * Store a resource by the hash of its contents.
+   *
+   * Content addressing is what keeps one screenshot taken twice, or a file
+   * downloaded in three scenarios, from being stored three times.
+   *
+   * @param {Buffer|string} contents - Resource bytes
+   * @param {string} [extension] - Extension to keep, such as `.png`
+   * @returns {Promise<Object|null>} `{member, sha256, bytes}`
+   */
+  async function writeArtifact(contents, extension = '') {
+    const data = Buffer.isBuffer(contents) ? contents : Buffer.from(contents);
+    const sha256 = createHash('sha256').update(data).digest('hex');
+    const member = `${TRACE_FILES.ARTIFACTS_DIR}/${sha256}${extension}`;
+
+    try {
+      await fs.access(path.join(root, member));
+      return { member, sha256, bytes: data.length, deduplicated: true };
+    } catch {
+      // Not stored yet.
+    }
+
+    const result = await writeMember(member, data);
+    return result ? { ...result, sha256, deduplicated: false } : null;
+  }
+
+  return {
+    root,
+    counts,
+    problems,
+    get dropped() {
+      return dropped;
+    },
+    get bytesWritten() {
+      return written;
+    },
+    appendEvent,
+    writeMember,
+    writeCheckpoint,
+    writeMutations,
+    writeArtifact,
+    drop,
+    /**
+     * Write the manifest and close the timeline.
+     *
+     * @param {Object} manifest - Manifest built by `createManifest`
+     * @returns {Promise<Object>} The manifest as written
+     */
+    close: async (manifest) => {
+      const finished = {
+        ...manifest,
+        counts: { ...counts },
+        dropped,
+        outcome:
+          dropped > 0 && manifest.outcome === TRACE_OUTCOME.COMPLETE
+            ? TRACE_OUTCOME.PARTIAL
+            : manifest.outcome,
+      };
+      // Anything a listener reported on its way out is still queued.
+      await appendQueue;
+      await eventsHandle.close();
+      await fs.writeFile(
+        path.join(root, TRACE_FILES.MANIFEST),
+        `${JSON.stringify(finished, null, 2)}\n`,
+        { mode: TRACE_FILE_MODE }
+      );
+      return finished;
+    },
+  };
+}