browser-commander 0.17.2 → 0.19.0

package/src/downloads/naming.js

@@ -0,0 +1,246 @@
+/**
+ * Naming and placement rules for managed downloads (issue #88).
+ *
+ * Everything here answers one question: given a name a *page* chose, where is
+ * it safe to write the bytes? A suggested filename is attacker-controlled
+ * input, so it is treated as a hint and never as a path.
+ */
+
+import path from 'node:path';
+
+/** The type a server sends when it cannot name the format either. */
+const GENERIC_BINARY_TYPE = 'application/octet-stream';
+
+/** Extensions derived from a declared or detected MIME type. */
+const MIME_EXTENSIONS = new Map([
+  ['application/pdf', '.pdf'],
+  ['application/json', '.json'],
+  ['application/zip', '.zip'],
+  ['application/gzip', '.gz'],
+  ['application/x-tar', '.tar'],
+  [GENERIC_BINARY_TYPE, '.bin'],
+  ['application/vnd.ms-excel', '.xls'],
+  [
+    'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+    '.xlsx',
+  ],
+  ['application/msword', '.doc'],
+  [
+    'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+    '.docx',
+  ],
+  ['text/csv', '.csv'],
+  ['text/html', '.html'],
+  ['text/plain', '.txt'],
+  ['image/png', '.png'],
+  ['image/jpeg', '.jpg'],
+  ['image/gif', '.gif'],
+  ['image/svg+xml', '.svg'],
+  ['image/webp', '.webp'],
+  ['video/mp4', '.mp4'],
+  ['audio/mpeg', '.mp3'],
+]);
+
+/** Leading bytes that identify a format regardless of what the page claimed. */
+const MAGIC_NUMBERS = [
+  { extension: '.pdf', bytes: [0x25, 0x50, 0x44, 0x46] },
+  { extension: '.png', bytes: [0x89, 0x50, 0x4e, 0x47] },
+  { extension: '.gif', bytes: [0x47, 0x49, 0x46, 0x38] },
+  { extension: '.jpg', bytes: [0xff, 0xd8, 0xff] },
+  { extension: '.zip', bytes: [0x50, 0x4b, 0x03, 0x04] },
+  { extension: '.gz', bytes: [0x1f, 0x8b] },
+];
+
+/** Name used when a page suggests nothing usable at all. */
+const FALLBACK_NAME = 'download';
+
+/** Windows device names that cannot be used as files even on other systems. */
+const RESERVED_NAMES = new Set([
+  'con',
+  'prn',
+  'aux',
+  'nul',
+  ...Array.from({ length: 9 }, (_, index) => `com${index + 1}`),
+  ...Array.from({ length: 9 }, (_, index) => `lpt${index + 1}`),
+]);
+
+/** A name that is only dots carries no information and is not a name. */
+const ONLY_DOTS = /^\.+$/;
+
+/** One whitespace character, tested one position at a time. */
+const WHITESPACE = /\s/;
+
+/**
+ * Trim the padding off a suggested name: leading whitespace, and trailing
+ * whitespace or dots.
+ *
+ * `/^\s+|[\s.]+$/` says the same thing in one line, but it backtracks
+ * quadratically over a long run of whitespace, and this string was chosen by
+ * the page. Walking in from both ends is one pass and cannot be made to cost
+ * more. Leading dots survive on purpose: `.bashrc` is a name, not padding.
+ *
+ * @param {string} value - Name to trim
+ * @returns {string} The name without its padding
+ */
+function trimNameEdges(value) {
+  let start = 0;
+  let end = value.length;
+
+  while (start < end && WHITESPACE.test(value[start])) {
+    start += 1;
+  }
+  while (
+    end > start &&
+    (value[end - 1] === '.' || WHITESPACE.test(value[end - 1]))
+  ) {
+    end -= 1;
+  }
+
+  return value.slice(start, end);
+}
+
+/**
+ * Strip a page-supplied name down to something that can only ever be a file
+ * inside the download root.
+ *
+ * Path separators, drive letters, control characters and `..` segments are
+ * removed rather than rejected, because a rejected download is a lost download
+ * and the caller asked for the file, not for the page's spelling of it.
+ *
+ * @param {string} suggested - Name suggested by the page or engine
+ * @returns {string} A single safe path segment
+ */
+export function sanitizeDownloadName(suggested) {
+  const raw = typeof suggested === 'string' ? suggested : '';
+
+  // Take the last segment under both separators: a page may suggest
+  // `../../etc/passwd` or `C:\Windows\system32\x`, and neither is a location
+  // we are willing to honor.
+  const lastSegment = raw.split(/[/\\]/).pop() ?? '';
+
+  const cleaned = trimNameEdges(
+    lastSegment
+      // eslint-disable-next-line no-control-regex -- control characters in a filename are exactly what we are removing
+      .replace(/[\u0000-\u001f\u007f]/g, '')
+      .replace(/[:*?"<>|]/g, '_')
+  );
+
+  if (!cleaned || ONLY_DOTS.test(cleaned)) {
+    return FALLBACK_NAME;
+  }
+
+  const stem = cleaned.slice(0, cleaned.lastIndexOf('.') + 1 || undefined);
+  if (RESERVED_NAMES.has(stem.replace(/\.$/, '').toLowerCase())) {
+    return `_${cleaned}`;
+  }
+
+  return cleaned;
+}
+
+/**
+ * Guess an extension from the first bytes of the file.
+ *
+ * @param {Buffer|Uint8Array|undefined} head - Leading bytes of the download
+ * @returns {string} Extension including the dot, or an empty string
+ */
+export function extensionFromContent(head) {
+  if (!head || head.length === 0) {
+    return '';
+  }
+
+  const found = MAGIC_NUMBERS.find(({ bytes }) =>
+    bytes.every((byte, index) => head[index] === byte)
+  );
+  return found ? found.extension : '';
+}
+
+/**
+ * Give a name an extension when the page did not supply one.
+ *
+ * Pages hand out UUID-like names constantly; a file called
+ * `7f1c...-9ab2` is unusable to a human and unopenable by the OS, so the
+ * declared MIME type (or the bytes themselves) supplies the missing suffix.
+ *
+ * @param {Object} options - Naming options
+ * @param {string} options.name - Sanitized name
+ * @param {string} [options.mimeType] - MIME type declared by the server
+ * @param {Buffer|Uint8Array} [options.head] - Leading bytes of the file
+ * @returns {string} Name with an extension when one could be determined
+ */
+export function withExtension({ name, mimeType, head }) {
+  if (path.extname(name)) {
+    return name;
+  }
+
+  const declaredType = String(mimeType ?? '')
+    .split(';')[0]
+    .trim()
+    .toLowerCase();
+  const declared = MIME_EXTENSIONS.get(declaredType);
+  const sniffed = extensionFromContent(head);
+  // `application/octet-stream` is what a server sends when it does not know
+  // either, so the bytes outrank it; any other declared type is a real claim.
+  const extension =
+    declaredType === GENERIC_BINARY_TYPE
+      ? sniffed || declared
+      : declared || sniffed;
+  return extension ? `${name}${extension}` : name;
+}
+
+/**
+ * Report whether a resolved path stays inside the download root.
+ *
+ * The check is done on resolved paths rather than on the name, so a symlinked
+ * or relative root cannot be used to step outside it.
+ *
+ * @param {string} root - Absolute download directory
+ * @param {string} candidate - Absolute candidate path
+ * @returns {boolean} Whether the candidate is inside the root
+ */
+export function isInsideRoot(root, candidate) {
+  const relative = path.relative(path.resolve(root), path.resolve(candidate));
+  return (
+    relative !== '' &&
+    !relative.startsWith('..') &&
+    !path.isAbsolute(relative) &&
+    !relative.split(path.sep).includes('..')
+  );
+}
+
+/**
+ * Resolve a safe absolute path for a download inside the root.
+ *
+ * @param {string} root - Absolute download directory
+ * @param {string} name - Sanitized file name
+ * @returns {string} Absolute path inside the root
+ * @throws {Error} When the name would escape the root
+ */
+export function resolveInsideRoot(root, name) {
+  const candidate = path.resolve(root, name);
+  if (!isInsideRoot(root, candidate)) {
+    throw new Error(
+      `refusing to write "${name}" outside the download directory ${root}`
+    );
+  }
+  return candidate;
+}
+
+/**
+ * Produce the candidate names a rename-on-conflict policy tries, in order.
+ *
+ * `report.pdf`, `report (2).pdf`, `report (3).pdf`, … — the order is fixed, so
+ * two runs of the same scenario produce the same names.
+ *
+ * @param {string} name - Sanitized file name
+ * @param {number} attempt - Zero-based attempt number
+ * @returns {string} Candidate name for that attempt
+ */
+export function renamedCandidate(name, attempt) {
+  if (attempt === 0) {
+    return name;
+  }
+
+  const extension = path.extname(name);
+  const stem = extension ? name.slice(0, -extension.length) : name;
+  return `${stem} (${attempt + 1})${extension}`;
+}

package/src/downloads/sources.js

@@ -0,0 +1,238 @@
+/**
+ * Engine-specific download event sources (issue #88).
+ *
+ * Each source turns one engine's download notifications into the same three
+ * calls - `started`, `finished`, `failed` - so the manager above them does not
+ * know or care whether the bytes came from Playwright or from CDP, and a
+ * download a human started by hand is reported exactly like an automated one.
+ */
+
+import path from 'node:path';
+import fs from 'node:fs/promises';
+
+import { ARTIFACT_DIRECTORY_MODE } from './destination.js';
+import {
+  DEFAULT_STAGING_POLL_INTERVAL,
+  DEFAULT_STAGING_TIMEOUT,
+  describeStagingTimeout,
+  waitForStagedFile,
+} from './staging.js';
+
+/** Why a download ended without a file. */
+export const DOWNLOAD_FAILURE = Object.freeze({
+  CANCELLED: 'cancelled',
+  FAILED: 'failed',
+});
+
+/** Directory the engine writes into before we place the file. */
+export const STAGING_DIRECTORY = '.browser-commander-staging';
+
+/**
+ * Classify an engine's failure text.
+ *
+ * "A download UI entry is not completion evidence" (issue #88): the engine's
+ * own words are the only thing that distinguishes a user cancelling a download
+ * from a network error, so they are preserved and classified, not flattened.
+ *
+ * @param {string} [reason] - Engine-reported reason
+ * @returns {string} One of {@link DOWNLOAD_FAILURE}
+ */
+export function classifyFailure(reason) {
+  return /cancel/i.test(String(reason ?? ''))
+    ? DOWNLOAD_FAILURE.CANCELLED
+    : DOWNLOAD_FAILURE.FAILED;
+}
+
+/**
+ * Attach to Playwright's context-level download event.
+ *
+ * @param {Object} options - Source options
+ * @param {Object} options.context - Playwright browser context
+ * @param {Object} options.sink - `{started, finished, failed}` callbacks
+ * @returns {Object} Handle with `detach()`
+ */
+export function attachPlaywrightSource({ context, sink }) {
+  const onDownload = async (download) => {
+    const record = sink.started({
+      engineHandle: download,
+      url: download.url(),
+      suggestedFilename: download.suggestedFilename(),
+      page: download.page?.(),
+    });
+
+    try {
+      const failure = await download.failure();
+      if (failure) {
+        sink.failed(record, classifyFailure(failure), failure);
+        return;
+      }
+
+      // `path()` resolves only once the bytes are on disk, which is the
+      // engine's own evidence that the download completed.
+      const enginePath = await download.path();
+      sink.finished(record, { path: enginePath });
+    } catch (error) {
+      sink.failed(record, classifyFailure(error.message), error.message);
+    }
+  };
+
+  context.on('download', onDownload);
+  return {
+    detach: () => context.off('download', onDownload),
+  };
+}
+
+/**
+ * Point Chromium at a staging directory and report every download it starts.
+ *
+ * This is the only source that sees a download a *person* started, because
+ * `Browser.setDownloadBehavior` is browser-wide rather than per-automation.
+ *
+ * @param {Object} options - Source options
+ * @param {Object} options.session - CDP session with Browser domain access
+ * @param {string} options.root - Managed download directory
+ * @param {Object} options.sink - `{started, finished, failed, track}` callbacks
+ * @param {number} [options.stagingTimeout] - Budget for the bytes to land on disk
+ * @param {number} [options.stagingPollInterval] - Milliseconds between readings
+ * @returns {Promise<Object>} Handle with `detach()` and `stagingDirectory`
+ */
+export async function attachCdpSource({
+  session,
+  root,
+  sink,
+  stagingTimeout,
+  stagingPollInterval,
+}) {
+  const stagingDirectory = path.join(root, STAGING_DIRECTORY);
+  await fs.mkdir(stagingDirectory, {
+    recursive: true,
+    mode: ARTIFACT_DIRECTORY_MODE,
+  });
+
+  // `allowAndName` writes each file under its GUID, so two downloads that
+  // suggest the same name cannot overwrite each other before we have placed
+  // them, and the GUID is the identity we deduplicate on.
+  await session.send('Browser.setDownloadBehavior', {
+    behavior: 'allowAndName',
+    downloadPath: stagingDirectory,
+    eventsEnabled: true,
+  });
+
+  const byGuid = new Map();
+
+  const onWillBegin = ({ guid, url, suggestedFilename }) => {
+    byGuid.set(
+      guid,
+      sink.started({
+        engineHandle: guid,
+        url,
+        suggestedFilename,
+      })
+    );
+  };
+
+  /**
+   * Publish a completed download once its staged bytes are readable.
+   *
+   * `state: "completed"` is the browser's word, not the filesystem's. Chromium
+   * writes `<guid>.crdownload` and renames it, so opening `<guid>` the instant
+   * the event arrives could - and did, in issue #92 - fail with `ENOENT` on a
+   * download that had in fact succeeded. Waiting for the file turns that race
+   * into either a saved artifact or a named failure, never a lost one.
+   *
+   * @param {Object} record - Artifact record
+   * @param {string} stagedPath - Path Chromium staged the bytes under
+   * @returns {Promise<void>}
+   */
+  const publishWhenStaged = async (record, stagedPath) => {
+    const budget = stagingTimeout ?? DEFAULT_STAGING_TIMEOUT;
+    const settled = await waitForStagedFile({
+      path: stagedPath,
+      timeout: budget,
+      interval: stagingPollInterval ?? DEFAULT_STAGING_POLL_INTERVAL,
+    });
+
+    if (!settled.ready) {
+      sink.failed(
+        record,
+        DOWNLOAD_FAILURE.FAILED,
+        describeStagingTimeout({
+          path: stagedPath,
+          timeout: budget,
+          reason: settled.reason,
+        })
+      );
+      return;
+    }
+
+    await sink.finished(record, { path: stagedPath, removeSource: true });
+  };
+
+  const onProgress = ({ guid, state }) => {
+    const record = byGuid.get(guid);
+    if (!record || state === 'inProgress') {
+      return;
+    }
+
+    byGuid.delete(guid);
+    if (state === 'completed') {
+      // Tracked before it is awaited: the wait is part of the download's
+      // lifecycle, so `idle()` and `dispose()` must not return while it runs.
+      sink.track(publishWhenStaged(record, path.join(stagingDirectory, guid)));
+      return;
+    }
+    sink.failed(
+      record,
+      DOWNLOAD_FAILURE.CANCELLED,
+      `download ${state} by the browser`
+    );
+  };
+
+  session.on('Browser.downloadWillBegin', onWillBegin);
+  session.on('Browser.downloadProgress', onProgress);
+
+  return {
+    stagingDirectory,
+    detach: () => {
+      session.off?.('Browser.downloadWillBegin', onWillBegin);
+      session.off?.('Browser.downloadProgress', onProgress);
+    },
+  };
+}
+
+/**
+ * Open a CDP session that can drive the Browser domain.
+ *
+ * Both engines can open a session that is not tied to a single page, which is
+ * what the Browser domain needs: one session then covers every page and every
+ * download no page started.
+ *
+ * @param {Object} options - Session options
+ * @param {string} options.engine - Engine name
+ * @param {Object} [options.browser] - Browser or persistent context
+ * @param {Object} [options.page] - Page to borrow a session from
+ * @returns {Promise<Object>|undefined} CDP session, or undefined when unavailable
+ */
+export function openBrowserCdpSession({ engine, browser, page }) {
+  if (engine === 'puppeteer') {
+    const target = browser?.target?.();
+    if (target?.createCDPSession) {
+      return target.createCDPSession();
+    }
+    return page?.target?.().createCDPSession?.();
+  }
+
+  // A page-scoped session only hears about the downloads that page started,
+  // so a browser-wide session is what makes a download a *person* began - in a
+  // tab the automation never opened - observable at all.
+  const browserHandle = browser?.browser?.() ?? browser;
+  if (browserHandle?.newBrowserCDPSession) {
+    return browserHandle.newBrowserCDPSession();
+  }
+
+  const context = browser?.browserContext?.() ?? browser;
+  if (page && context?.newCDPSession) {
+    return context.newCDPSession(page);
+  }
+  return undefined;
+}

package/src/downloads/staging.js

@@ -0,0 +1,146 @@
+/**
+ * Waiting for a staged download to actually be on disk (issue #92).
+ *
+ * `Browser.downloadProgress` with `state: "completed"` is the browser saying it
+ * has stopped downloading, not the filesystem saying the bytes are there under
+ * the name we are about to open. Chromium writes into `<guid>.crdownload` and
+ * renames it, and the event can reach us before that rename is visible - which
+ * is how a completed download became
+ * `ENOENT: no such file or directory, open '<root>/.browser-commander-staging/<guid>'`.
+ *
+ * This module turns "the browser said completed" into "the file is readable and
+ * has stopped growing", within a bounded budget. Running out of that budget is
+ * an explicit failure, never a silent success: issue #88's rule that "a
+ * download UI entry is not completion evidence" is exactly the rule being
+ * applied one level lower down.
+ */
+
+import fs from 'node:fs/promises';
+import { setTimeout as delay } from 'node:timers/promises';
+
+import { createDeadline } from '../core/readiness.js';
+
+/** How long to wait for a completed download's bytes to appear, by default. */
+export const DEFAULT_STAGING_TIMEOUT = 10000;
+
+/** How often the staged file is looked at while waiting. */
+export const DEFAULT_STAGING_POLL_INTERVAL = 25;
+
+/** Suffixes Chromium uses for a file it is still writing. */
+export const STAGING_IN_PROGRESS_SUFFIXES = Object.freeze([
+  '.crdownload',
+  '.partial',
+]);
+
+/**
+ * Look at a staged file once.
+ *
+ * @param {string} filePath - Path the engine staged the download under
+ * @param {string[]} suffixes - Suffixes that mark a file still being written
+ * @returns {Promise<{size: number|null, reason: string|null}>} What was seen
+ */
+async function observeStagedFile(filePath, suffixes) {
+  for (const suffix of suffixes) {
+    try {
+      await fs.stat(`${filePath}${suffix}`);
+      return { size: null, reason: `a ${suffix} file is still being written` };
+    } catch {
+      // Absent is what we want: the rename to the final staged name has
+      // already happened, or Chromium never used this suffix at all.
+    }
+  }
+
+  let handle;
+  try {
+    // Opening rather than stat'ing is deliberate: "readable" is the property
+    // `saveDownload()` needs, and a name that resolves is not the same thing
+    // as a file this process is allowed to read.
+    handle = await fs.open(filePath, 'r');
+  } catch (error) {
+    return {
+      size: null,
+      reason:
+        error.code === 'ENOENT'
+          ? 'the file has not appeared yet'
+          : `the file could not be opened: ${error.message}`,
+    };
+  }
+
+  try {
+    const stats = await handle.stat();
+    return { size: stats.size, reason: null };
+  } finally {
+    await handle.close();
+  }
+}
+
+/**
+ * Wait until a staged download is readable and has stopped growing.
+ *
+ * Stability is decided by two consecutive readings of the same size, the same
+ * evidence the Selenium directory watcher uses: a file that merely exists is
+ * not evidence that its last byte was written.
+ *
+ * @param {Object} options - Wait options
+ * @param {string} options.path - Path the engine staged the download under
+ * @param {number} [options.timeout] - Total budget in milliseconds
+ * @param {number} [options.interval] - Milliseconds between readings
+ * @param {string[]} [options.inProgressSuffixes] - Suffixes of a partial file
+ * @returns {Promise<{ready: boolean, bytes: number|null, reason: string|null, waitedMs: number}>} Outcome
+ */
+export async function waitForStagedFile(options = {}) {
+  const {
+    path: filePath,
+    timeout = DEFAULT_STAGING_TIMEOUT,
+    interval = DEFAULT_STAGING_POLL_INTERVAL,
+    inProgressSuffixes = STAGING_IN_PROGRESS_SUFFIXES,
+  } = options;
+
+  const deadline = createDeadline({ timeout });
+  let previousSize;
+  // Always set by the reading below before the deadline can be checked.
+  let reason;
+
+  for (;;) {
+    const seen = await observeStagedFile(filePath, inProgressSuffixes);
+    if (seen.reason !== null) {
+      reason = seen.reason;
+      // A file that vanished has to prove itself stable again from scratch.
+      previousSize = undefined;
+    } else if (seen.size === previousSize) {
+      return {
+        ready: true,
+        bytes: seen.size,
+        reason: null,
+        waitedMs: deadline.elapsedMs(),
+      };
+    } else {
+      previousSize = seen.size;
+      reason = `the file was still growing at ${seen.size} bytes`;
+    }
+
+    const remaining = deadline.remainingMs();
+    if (remaining === 0) {
+      return {
+        ready: false,
+        bytes: null,
+        reason,
+        waitedMs: deadline.elapsedMs(),
+      };
+    }
+    await delay(Math.min(interval, remaining));
+  }
+}
+
+/**
+ * The failure text a staged download that never materialized is reported under.
+ *
+ * @param {Object} options - Failure details
+ * @param {string} options.path - Path the engine staged the download under
+ * @param {number} options.timeout - Budget that was exhausted
+ * @param {string} options.reason - What the last reading saw
+ * @returns {string} Message kept verbatim on the artifact
+ */
+export function describeStagingTimeout({ path: filePath, timeout, reason }) {
+  return `the browser reported the download as completed, but ${filePath} was not readable within ${timeout}ms: ${reason}`;
+}