browser-commander 0.17.2 → 0.18.0

package/src/downloads/sources.js

@@ -0,0 +1,188 @@
+/**
+ * Engine-specific download event sources (issue #88).
+ *
+ * Each source turns one engine's download notifications into the same three
+ * calls - `started`, `finished`, `failed` - so the manager above them does not
+ * know or care whether the bytes came from Playwright or from CDP, and a
+ * download a human started by hand is reported exactly like an automated one.
+ */
+
+import path from 'node:path';
+import fs from 'node:fs/promises';
+
+import { ARTIFACT_DIRECTORY_MODE } from './destination.js';
+
+/** Why a download ended without a file. */
+export const DOWNLOAD_FAILURE = Object.freeze({
+  CANCELLED: 'cancelled',
+  FAILED: 'failed',
+});
+
+/** Directory the engine writes into before we place the file. */
+export const STAGING_DIRECTORY = '.browser-commander-staging';
+
+/**
+ * Classify an engine's failure text.
+ *
+ * "A download UI entry is not completion evidence" (issue #88): the engine's
+ * own words are the only thing that distinguishes a user cancelling a download
+ * from a network error, so they are preserved and classified, not flattened.
+ *
+ * @param {string} [reason] - Engine-reported reason
+ * @returns {string} One of {@link DOWNLOAD_FAILURE}
+ */
+export function classifyFailure(reason) {
+  return /cancel/i.test(String(reason ?? ''))
+    ? DOWNLOAD_FAILURE.CANCELLED
+    : DOWNLOAD_FAILURE.FAILED;
+}
+
+/**
+ * Attach to Playwright's context-level download event.
+ *
+ * @param {Object} options - Source options
+ * @param {Object} options.context - Playwright browser context
+ * @param {Object} options.sink - `{started, finished, failed}` callbacks
+ * @returns {Object} Handle with `detach()`
+ */
+export function attachPlaywrightSource({ context, sink }) {
+  const onDownload = async (download) => {
+    const record = sink.started({
+      engineHandle: download,
+      url: download.url(),
+      suggestedFilename: download.suggestedFilename(),
+      page: download.page?.(),
+    });
+
+    try {
+      const failure = await download.failure();
+      if (failure) {
+        sink.failed(record, classifyFailure(failure), failure);
+        return;
+      }
+
+      // `path()` resolves only once the bytes are on disk, which is the
+      // engine's own evidence that the download completed.
+      const enginePath = await download.path();
+      sink.finished(record, { path: enginePath });
+    } catch (error) {
+      sink.failed(record, classifyFailure(error.message), error.message);
+    }
+  };
+
+  context.on('download', onDownload);
+  return {
+    detach: () => context.off('download', onDownload),
+  };
+}
+
+/**
+ * Point Chromium at a staging directory and report every download it starts.
+ *
+ * This is the only source that sees a download a *person* started, because
+ * `Browser.setDownloadBehavior` is browser-wide rather than per-automation.
+ *
+ * @param {Object} options - Source options
+ * @param {Object} options.session - CDP session with Browser domain access
+ * @param {string} options.root - Managed download directory
+ * @param {Object} options.sink - `{started, finished, failed}` callbacks
+ * @returns {Promise<Object>} Handle with `detach()` and `stagingDirectory`
+ */
+export async function attachCdpSource({ session, root, sink }) {
+  const stagingDirectory = path.join(root, STAGING_DIRECTORY);
+  await fs.mkdir(stagingDirectory, {
+    recursive: true,
+    mode: ARTIFACT_DIRECTORY_MODE,
+  });
+
+  // `allowAndName` writes each file under its GUID, so two downloads that
+  // suggest the same name cannot overwrite each other before we have placed
+  // them, and the GUID is the identity we deduplicate on.
+  await session.send('Browser.setDownloadBehavior', {
+    behavior: 'allowAndName',
+    downloadPath: stagingDirectory,
+    eventsEnabled: true,
+  });
+
+  const byGuid = new Map();
+
+  const onWillBegin = ({ guid, url, suggestedFilename }) => {
+    byGuid.set(
+      guid,
+      sink.started({
+        engineHandle: guid,
+        url,
+        suggestedFilename,
+      })
+    );
+  };
+
+  const onProgress = ({ guid, state }) => {
+    const record = byGuid.get(guid);
+    if (!record || state === 'inProgress') {
+      return;
+    }
+
+    byGuid.delete(guid);
+    if (state === 'completed') {
+      sink.finished(record, {
+        path: path.join(stagingDirectory, guid),
+        removeSource: true,
+      });
+      return;
+    }
+    sink.failed(
+      record,
+      DOWNLOAD_FAILURE.CANCELLED,
+      `download ${state} by the browser`
+    );
+  };
+
+  session.on('Browser.downloadWillBegin', onWillBegin);
+  session.on('Browser.downloadProgress', onProgress);
+
+  return {
+    stagingDirectory,
+    detach: () => {
+      session.off?.('Browser.downloadWillBegin', onWillBegin);
+      session.off?.('Browser.downloadProgress', onProgress);
+    },
+  };
+}
+
+/**
+ * Open a CDP session that can drive the Browser domain.
+ *
+ * Both engines can open a session that is not tied to a single page, which is
+ * what the Browser domain needs: one session then covers every page and every
+ * download no page started.
+ *
+ * @param {Object} options - Session options
+ * @param {string} options.engine - Engine name
+ * @param {Object} [options.browser] - Browser or persistent context
+ * @param {Object} [options.page] - Page to borrow a session from
+ * @returns {Promise<Object>|undefined} CDP session, or undefined when unavailable
+ */
+export function openBrowserCdpSession({ engine, browser, page }) {
+  if (engine === 'puppeteer') {
+    const target = browser?.target?.();
+    if (target?.createCDPSession) {
+      return target.createCDPSession();
+    }
+    return page?.target?.().createCDPSession?.();
+  }
+
+  // A page-scoped session only hears about the downloads that page started,
+  // so a browser-wide session is what makes a download a *person* began - in a
+  // tab the automation never opened - observable at all.
+  const browserHandle = browser?.browser?.() ?? browser;
+  if (browserHandle?.newBrowserCDPSession) {
+    return browserHandle.newBrowserCDPSession();
+  }
+
+  const context = browser?.browserContext?.() ?? browser;
+  if (page && context?.newCDPSession) {
+    return context.newCDPSession(page);
+  }
+  return undefined;
+}

package/src/downloads/store.js

@@ -0,0 +1,256 @@
+/**
+ * Writing a download into the managed directory (issue #88).
+ *
+ * The rule the whole module exists to keep: a file that appears under its
+ * final name is complete and has passed validation. Everything else lives
+ * under a `.partial` name and is removed.
+ */
+
+import { createHash } from 'node:crypto';
+import { createReadStream, createWriteStream } from 'node:fs';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { pipeline } from 'node:stream/promises';
+
+import { ARTIFACT_DIRECTORY_MODE, ARTIFACT_FILE_MODE } from './destination.js';
+import {
+  renamedCandidate,
+  resolveInsideRoot,
+  sanitizeDownloadName,
+  withExtension,
+} from './naming.js';
+
+/** How a name that is already taken is resolved. */
+export const DOWNLOAD_CONFLICT = Object.freeze({
+  RENAME: 'rename',
+  OVERWRITE: 'overwrite',
+  ERROR: 'error',
+});
+
+/** How many bytes are kept to sniff a format from. */
+const MAGIC_BYTES = 8;
+
+/** Highest rename attempt before giving up rather than looping forever. */
+const MAX_RENAME_ATTEMPTS = 1000;
+
+/**
+ * Open the download's bytes as a readable stream.
+ *
+ * @param {Object} source - Either `{stream}` or `{path}`
+ * @returns {Object} Readable stream
+ */
+function openSource(source) {
+  if (source.stream) {
+    return source.stream;
+  }
+  if (source.path) {
+    return createReadStream(source.path);
+  }
+  throw new TypeError('a download source needs either a stream or a path');
+}
+
+/**
+ * Stream the download into a partial file, hashing it on the way through.
+ *
+ * Hashing during the copy means the bytes are read once: reading the file a
+ * second time to checksum it would double the I/O and leave a window where the
+ * file could change between the two reads.
+ *
+ * @param {Object} options - Copy options
+ * @param {Object} options.source - `{stream}` or `{path}` to read from
+ * @param {string} options.partialPath - Where the bytes are written
+ * @returns {Promise<{bytes: number, checksum: string, head: Buffer}>} What was written
+ */
+async function copyToPartial({ source, partialPath }) {
+  const hash = createHash('sha256');
+  let bytes = 0;
+  let head = Buffer.alloc(0);
+
+  const readable = openSource(source);
+  const writable = createWriteStream(partialPath, { mode: ARTIFACT_FILE_MODE });
+
+  await pipeline(
+    readable,
+    async function* (chunks) {
+      for await (const chunk of chunks) {
+        hash.update(chunk);
+        bytes += chunk.length;
+        if (head.length < MAGIC_BYTES) {
+          head = Buffer.concat([head, chunk.subarray(0, MAGIC_BYTES)]).subarray(
+            0,
+            MAGIC_BYTES
+          );
+        }
+        yield chunk;
+      }
+    },
+    writable
+  );
+
+  return { bytes, checksum: hash.digest('hex'), head };
+}
+
+/**
+ * Report whether a path already exists.
+ *
+ * @param {string} candidate - Absolute path
+ * @returns {Promise<boolean>} Whether something is there
+ */
+async function exists(candidate) {
+  try {
+    await fs.access(candidate);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Choose the final path for a completed download.
+ *
+ * @param {Object} options - Placement options
+ * @param {string} options.root - Absolute download directory
+ * @param {string} options.name - Sanitized file name
+ * @param {string} options.conflict - Conflict policy
+ * @returns {Promise<string>} Absolute final path
+ * @throws {Error} When the name is taken and the policy says to fail
+ */
+export async function resolveFinalPath({ root, name, conflict }) {
+  if (conflict === DOWNLOAD_CONFLICT.OVERWRITE) {
+    return resolveInsideRoot(root, name);
+  }
+
+  for (let attempt = 0; attempt < MAX_RENAME_ATTEMPTS; attempt += 1) {
+    const candidate = resolveInsideRoot(root, renamedCandidate(name, attempt));
+    if (!(await exists(candidate))) {
+      return candidate;
+    }
+    if (conflict === DOWNLOAD_CONFLICT.ERROR) {
+      throw new Error(
+        `refusing to replace ${candidate}: downloads.conflict is 'error'`
+      );
+    }
+  }
+
+  throw new Error(
+    `could not find a free name for "${name}" after ${MAX_RENAME_ATTEMPTS} attempts`
+  );
+}
+
+/**
+ * Save a download into the managed directory.
+ *
+ * @param {Object} options - Save options
+ * @param {string} options.root - Absolute download directory
+ * @param {Object} options.source - `{stream}` or `{path}` to read from
+ * @param {string} options.suggestedFilename - Name the page suggested
+ * @param {string} [options.mimeType] - MIME type declared by the server
+ * @param {string} [options.conflict] - Conflict policy
+ * @param {Function} [options.filename] - Caller naming callback
+ * @param {Function} [options.validate] - Validation run before the file is
+ *   published; throwing or returning `false` rejects the download
+ * @returns {Promise<{path: string, bytes: number, checksum: string}>} What was saved
+ */
+export async function saveDownload({
+  root,
+  source,
+  suggestedFilename,
+  mimeType,
+  conflict = DOWNLOAD_CONFLICT.RENAME,
+  filename,
+  validate,
+}) {
+  await fs.mkdir(root, { recursive: true, mode: ARTIFACT_DIRECTORY_MODE });
+
+  // A caller callback runs *before* sanitization, never instead of it: it is a
+  // naming preference, not a grant of write access outside the root.
+  const chosen = filename
+    ? await filename({ suggestedFilename, mimeType })
+    : suggestedFilename;
+  const safeName = sanitizeDownloadName(chosen ?? suggestedFilename);
+
+  const partialPath = resolveInsideRoot(
+    root,
+    `${safeName}.${process.pid}.${Date.now()}.partial`
+  );
+
+  let written;
+  try {
+    written = await copyToPartial({ source, partialPath });
+
+    if (validate) {
+      // Validation sees the partial file, so a rejected download never exists
+      // under the name a caller would pick it up by.
+      const verdict = await validate({
+        path: partialPath,
+        bytes: written.bytes,
+        checksum: written.checksum,
+        mimeType,
+        suggestedFilename,
+      });
+      // A predicate that returns `false` rejects just as loudly as one that
+      // throws; anything else (including `undefined`) accepts, so a validator
+      // written only for its side effects still works.
+      if (verdict === false) {
+        throw new Error(
+          `"${safeName}" was rejected by the caller's validation`
+        );
+      }
+    }
+  } catch (error) {
+    await fs.rm(partialPath, { force: true });
+    throw error;
+  }
+
+  const finalName = withExtension({
+    name: safeName,
+    mimeType,
+    head: written.head,
+  });
+
+  try {
+    const finalPath = await resolveFinalPath({
+      root,
+      name: finalName,
+      conflict,
+    });
+    // Rename rather than copy: within one filesystem it is atomic, so a reader
+    // watching the directory never sees a half-written file under this name.
+    await fs.rename(partialPath, finalPath);
+    await fs.chmod(finalPath, ARTIFACT_FILE_MODE);
+
+    return {
+      path: finalPath,
+      bytes: written.bytes,
+      checksum: written.checksum,
+    };
+  } catch (error) {
+    await fs.rm(partialPath, { force: true });
+    throw error;
+  }
+}
+
+/**
+ * Remove every partial file left behind in a download directory.
+ *
+ * @param {string} root - Absolute download directory
+ * @returns {Promise<string[]>} Paths that were removed
+ */
+export async function cleanPartials(root) {
+  let entries;
+  try {
+    entries = await fs.readdir(root);
+  } catch {
+    return [];
+  }
+
+  const removed = [];
+  for (const entry of entries) {
+    if (entry.endsWith('.partial')) {
+      const candidate = path.join(root, entry);
+      await fs.rm(candidate, { force: true });
+      removed.push(candidate);
+    }
+  }
+  return removed;
+}

package/src/exports.js

@@ -60,6 +60,7 @@ export {
   setContent,
   waitForNavigation,
   waitForPageReady,
+  waitForReady,
   waitAfterAction,
   // Navigation verification
   defaultNavigationVerification,
@@ -115,8 +116,31 @@ export {
   defaultClickVerification,
   capturePreClickState,
   verifyClick,
+  // Truthful click result model
+  CLICK_STATUS,
+  CLICK_EFFECT,
+  // Orthogonal activation options
+  CLICK_ACTIVATION,
+  CLICK_SCROLL,
+  CLICK_ACTIONABILITY,
+  ScrollConstraintError,
 } from './interactions/click.js';
 
+export {
+  READINESS_STATUS,
+  LONG_LIVED_REQUEST_PATTERNS,
+  isLongLivedRequest,
+  createDeadline,
+  runReadinessChecks,
+  // Composable readiness checks
+  urlStableFor,
+  networkIdleFor,
+  domStableFor,
+  visibleImages,
+  predicate,
+  stableCheck,
+} from './core/readiness.js';
+
 export {
   checkIfElementEmpty,
   performFill,
@@ -173,3 +197,51 @@ export {
   findFingerprintLimitation,
   relevantFingerprintLimitations,
 } from './fingerprint/limitations.js';
+
+// Re-export the managed download subsystem: one lifecycle for automated and
+// manual downloads, with files that outlive the browser (issue #88).
+export {
+  attachDownloads,
+  normalizeDownloadOptions,
+} from './downloads/attach.js';
+export { DOWNLOAD_EVENT, createDownloadManager } from './downloads/manager.js';
+export {
+  ARTIFACT_DIRECTORY_MODE,
+  ARTIFACT_FILE_MODE,
+  DOWNLOAD_DIRECTORY_PRESETS,
+  prepareDownloadDirectory,
+  resolveDownloadDirectory,
+} from './downloads/destination.js';
+export {
+  DOWNLOAD_CONFLICT,
+  cleanPartials,
+  saveDownload,
+} from './downloads/store.js';
+export { DOWNLOAD_FAILURE } from './downloads/sources.js';
+export {
+  extensionFromContent,
+  sanitizeDownloadName,
+  withExtension,
+} from './downloads/naming.js';
+
+// Re-export the portable trace subsystem: one recorder, one versioned bundle,
+// and one offline viewer for every consumer (issue #87).
+export { startTrace } from './traces/recorder.js';
+export {
+  TRACE_EVENT,
+  TRACE_EVENT_SOURCES,
+  TRACE_FILES,
+  TRACE_MODE,
+  TRACE_OUTCOME,
+  TRACE_SCHEMA_VERSION,
+} from './traces/schema.js';
+export { diffControlState, parseNdjson, readTrace } from './traces/reader.js';
+export { renderTraceViewer, writeTraceViewer } from './traces/viewer.js';
+export {
+  DEFAULT_REDACT_ATTRIBUTES,
+  DEFAULT_REDACT_SELECTORS,
+  REDACTED,
+  normalizePrivacyOptions,
+  redactUrl,
+  redactValue,
+} from './traces/redaction.js';

package/src/factory.js

@@ -20,6 +20,8 @@ import {
   notCondition,
 } from './core/page-trigger-manager.js';
 import { createBoundFunctions } from './bindings.js';
+import { attachDownloads } from './downloads/attach.js';
+import { startTrace } from './traces/recorder.js';
 
 /**
  * Create a browser commander instance for a specific page
@@ -28,6 +30,7 @@ import { createBoundFunctions } from './bindings.js';
  * @param {boolean} options.verbose - Enable verbose logging
  * @param {boolean} options.enableNetworkTracking - Enable network request tracking (default: true)
  * @param {boolean} options.enableNavigationManager - Enable navigation manager (default: true)
+ * @param {Object} [options.downloads] - A download manager to expose as commander.downloads, such as the one launchBrowser() returned
  * @returns {Object} - Browser commander API
  */
 export function makeBrowserCommander(options = {}) {
@@ -37,6 +40,7 @@ export function makeBrowserCommander(options = {}) {
     enableNetworkTracking = true,
     enableNavigationManager = true,
     enableDialogManager = true,
+    downloads = null,
   } = options;
 
   if (!page) {
@@ -108,6 +112,12 @@ export function makeBrowserCommander(options = {}) {
 
   // Cleanup function
   const destroy = async () => {
+    // Downloads are disposed first: disposing waits for bytes still being
+    // written, and a caller that destroys the commander right after a click
+    // should still find the file.
+    if (commander.downloads) {
+      await commander.downloads.dispose();
+    }
     if (pageTriggerManager) {
       await pageTriggerManager.destroy();
     }
@@ -142,6 +152,40 @@ export function makeBrowserCommander(options = {}) {
     // All bound functions
     ...boundFunctions,
 
+    // Managed downloads (issue #88). Present once a manager is attached,
+    // either by passing the one launchBrowser() returned or by calling
+    // configureDownloads() on an existing commander.
+    downloads,
+
+    /**
+     * Attach a managed download lifecycle to this commander's browser.
+     *
+     * @param {boolean|Object} [downloadOptions=true] - true for defaults, or {directory, persist, conflict}
+     * @returns {Promise<Object>} The attached download manager
+     */
+    configureDownloads: async (downloadOptions = true) => {
+      // Replacing a manager detaches the old one rather than leaving two
+      // sources writing into the same directory.
+      await commander.downloads?.dispose();
+      commander.downloads = await attachDownloads({
+        engine,
+        browser: page.context?.() ?? page.browser?.(),
+        page,
+        downloads: downloadOptions,
+        log,
+      });
+      return commander.downloads;
+    },
+
+    /**
+     * Start recording a privacy-aware trace of this session (issue #87).
+     *
+     * @param {Object} [traceOptions] - {output, mode, screenshots, dom, events, privacy, limits}
+     * @returns {Promise<Object>} The running trace: {checkpoint, event, stop}
+     */
+    startTrace: (traceOptions = {}) =>
+      startTrace({ commander, page, log, ...traceOptions }),
+
     // Lifecycle
     destroy,
 
@@ -209,6 +253,17 @@ export function makeBrowserCommander(options = {}) {
           );
         },
 
+    // Watch dialogs without answering them (issue #87): an observer is told
+    // about every dialog and never suppresses the manager's auto-dismissal.
+    observeDialogs: dialogManager
+      ? (fn) => dialogManager.observeDialogs(fn)
+      : () => {
+          throw new Error('observeDialogs requires enableDialogManager: true');
+        },
+    unobserveDialogs: dialogManager
+      ? (fn) => dialogManager.unobserveDialogs(fn)
+      : () => {},
+
     // URL condition helpers
     makeUrlCondition,
     allConditions,