browser-commander 0.17.2 → 0.18.0

package/src/downloads/manager.js

@@ -0,0 +1,453 @@
+/**
+ * Managed, persistent downloads (issue #88).
+ *
+ * The manager owns one lifecycle for every download the browser performs -
+ * automated or started by a person - and guarantees three things a caller
+ * cannot get from an engine event alone:
+ *
+ * - the file survives the page, context and browser that produced it;
+ * - a download is saved once and reported once, even when a global listener
+ *   and an awaited `capture()` both see it;
+ * - a file under its final name is complete and has passed validation.
+ */
+
+import { EventEmitter } from 'node:events';
+import fs from 'node:fs/promises';
+
+import { createDeadline } from '../core/readiness.js';
+import {
+  prepareDownloadDirectory,
+  resolveDownloadDirectory,
+} from './destination.js';
+import {
+  attachCdpSource,
+  attachPlaywrightSource,
+  classifyFailure,
+  DOWNLOAD_FAILURE,
+  openBrowserCdpSession,
+} from './sources.js';
+import { DOWNLOAD_CONFLICT, saveDownload } from './store.js';
+
+/** Lifecycle events every download passes through. */
+export const DOWNLOAD_EVENT = Object.freeze({
+  STARTED: 'started',
+  COMPLETED: 'completed',
+  FAILED: 'failed',
+  CANCELLED: 'cancelled',
+});
+
+/** How long `capture()` waits for a download by default. */
+export const DEFAULT_CAPTURE_TIMEOUT = 30000;
+
+/** Monotonic part of a download ID, so IDs are stable and ordered. */
+let downloadCounter = 0;
+
+/**
+ * Build the identifier a download is reported under.
+ *
+ * @returns {string} Stable download ID
+ */
+function nextDownloadId() {
+  downloadCounter += 1;
+  return `dl-${String(downloadCounter).padStart(6, '0')}`;
+}
+
+/**
+ * Find the object Playwright emits its `download` event on.
+ *
+ * A download is a *context*-level event. `launchBrowser()` hands back a
+ * persistent context, so the browser handle works there by accident; a
+ * `Browser` returned by `connectBrowser()` never emits `download` at all.
+ * Taking the page's context is what makes an attached browser behave exactly
+ * like a launched one.
+ *
+ * @param {Object} options - Handles the caller supplied
+ * @param {Object} [options.browser] - Browser or persistent context
+ * @param {Object} [options.page] - A page belonging to the browser
+ * @returns {Object|undefined} The context downloads are observed on
+ */
+function playwrightDownloadContext({ browser, page }) {
+  const fromPage =
+    typeof page?.context === 'function' ? page.context() : undefined;
+  return fromPage ?? browser;
+}
+
+/**
+ * Create the download manager for a browser.
+ *
+ * @param {Object} options - Manager options
+ * @param {string} options.engine - 'playwright' or 'puppeteer'
+ * @param {Object} [options.browser] - Browser or persistent context
+ * @param {Object} [options.context] - Playwright browser context
+ * @param {Object} [options.page] - A page belonging to the browser
+ * @param {string} [options.directory] - Absolute path, 'user-downloads' or 'temporary'
+ * @param {boolean} [options.persist=true] - Keep files after the browser closes
+ * @param {string} [options.conflict='rename'] - 'rename', 'overwrite' or 'error'
+ * @param {Function} [options.filename] - Naming callback for every download
+ * @param {Function} [options.validate] - Validation for every download
+ * @param {Object} [options.log] - Logger
+ * @returns {Promise<Object>} Download manager
+ */
+export async function createDownloadManager(options = {}) {
+  const {
+    engine = 'playwright',
+    browser,
+    context: explicitContext,
+    page,
+    directory,
+    persist = true,
+    conflict = DOWNLOAD_CONFLICT.RENAME,
+    filename,
+    validate,
+    log,
+  } = options;
+
+  const context =
+    explicitContext ??
+    (engine === 'playwright'
+      ? playwrightDownloadContext({ browser, page })
+      : browser);
+
+  // Resolved and probed before a single download can start: a permission
+  // problem found afterwards looks like a file that never arrived.
+  const root = await prepareDownloadDirectory(
+    await resolveDownloadDirectory(directory)
+  );
+
+  const emitter = new EventEmitter();
+  const artifacts = [];
+  const pending = new Map();
+  const waiters = new Set();
+  const seen = new Set();
+  const published = new Set();
+  const inFlight = new Set();
+
+  /**
+   * Hand a settled artifact to every waiter armed before it started.
+   *
+   * @param {Object} artifact - Settled artifact
+   * @returns {void}
+   */
+  const settleWaiters = (artifact) => {
+    for (const waiter of [...waiters]) {
+      if (waiter.accepts(artifact)) {
+        waiters.delete(waiter);
+        waiter.settle(artifact);
+      }
+    }
+  };
+
+  /**
+   * Publish an artifact under its lifecycle event exactly once.
+   *
+   * @param {Object} artifact - Artifact to publish
+   * @param {string} event - Lifecycle event name
+   * @returns {void}
+   */
+  const publish = (artifact, event) => {
+    // A download reaches its end once. Both a global listener and an awaited
+    // `capture()` read from this same record, so publishing twice would mean
+    // one file reported as two.
+    if (published.has(artifact)) {
+      return;
+    }
+    published.add(artifact);
+    artifact.state = event;
+    emitter.emit(event, artifact);
+    settleWaiters(artifact);
+  };
+
+  const sink = {
+    /**
+     * Record a download the engine has just announced.
+     *
+     * @param {Object} start - Engine-reported start details
+     * @returns {Object} Artifact record
+     */
+    started(start) {
+      // One engine download can be announced twice - a context listener and a
+      // page listener see the same object - so the engine's own handle is the
+      // identity, not the order events arrived in.
+      if (seen.has(start.engineHandle)) {
+        return pending.get(start.engineHandle);
+      }
+      seen.add(start.engineHandle);
+
+      const artifact = {
+        id: nextDownloadId(),
+        url: start.url,
+        suggestedFilename: start.suggestedFilename,
+        state: DOWNLOAD_EVENT.STARTED,
+        startedAt: new Date().toISOString(),
+        completedAt: null,
+        path: null,
+        mimeType: start.mimeType ?? null,
+        bytes: null,
+        checksum: null,
+        failure: null,
+      };
+
+      pending.set(start.engineHandle, artifact);
+      artifacts.push(artifact);
+      emitter.emit(DOWNLOAD_EVENT.STARTED, artifact);
+      return artifact;
+    },
+
+    /**
+     * Save a download whose bytes the engine has finished writing.
+     *
+     * Engine events are fire-and-forget, so the save is registered as in-flight
+     * work: otherwise `dispose()` could return while bytes are still being
+     * written and leave a `.partial` file as the only trace of the download.
+     *
+     * @param {Object} artifact - Artifact record
+     * @param {Object} source - `{path}` or `{stream}` to read the bytes from
+     * @returns {Promise<void>} Resolves once the artifact has been published
+     */
+    finished(artifact, source) {
+      const work = save(artifact, source);
+      inFlight.add(work);
+      return work.finally(() => inFlight.delete(work));
+    },
+
+    /**
+     * Record a download that ended without a file.
+     *
+     * @param {Object} artifact - Artifact record
+     * @param {string} kind - {@link DOWNLOAD_FAILURE} classification
+     * @param {string} reason - Engine-reported reason, kept verbatim
+     * @returns {void}
+     */
+    failed(artifact, kind, reason) {
+      if (!artifact) {
+        return;
+      }
+      artifact.failure = reason;
+      publish(
+        artifact,
+        kind === DOWNLOAD_FAILURE.CANCELLED
+          ? DOWNLOAD_EVENT.CANCELLED
+          : DOWNLOAD_EVENT.FAILED
+      );
+    },
+  };
+
+  /**
+   * Place a finished download's bytes under their final name.
+   *
+   * @param {Object} artifact - Artifact record
+   * @param {Object} source - `{path}` or `{stream}` to read the bytes from
+   * @returns {Promise<void>}
+   */
+  async function save(artifact, source) {
+    if (!artifact || artifact.state !== DOWNLOAD_EVENT.STARTED) {
+      return;
+    }
+
+    try {
+      const saved = await saveDownload({
+        root,
+        source,
+        suggestedFilename: artifact.suggestedFilename,
+        mimeType: artifact.mimeType,
+        conflict: artifact.conflict ?? conflict,
+        filename: artifact.filename ?? filename,
+        validate: artifact.validate ?? validate,
+      });
+
+      Object.assign(artifact, saved, {
+        completedAt: new Date().toISOString(),
+      });
+      publish(artifact, DOWNLOAD_EVENT.COMPLETED);
+    } catch (error) {
+      artifact.failure = error.message;
+      publish(artifact, DOWNLOAD_EVENT.FAILED);
+      log?.warn?.(`download ${artifact.id} could not be saved`);
+    } finally {
+      if (source.removeSource && source.path) {
+        await fs.rm(source.path, { force: true });
+      }
+    }
+  }
+
+  /**
+   * Wait until every download the manager has seen has been placed.
+   *
+   * @returns {Promise<void>}
+   */
+  const idle = async () => {
+    while (inFlight.size > 0) {
+      await Promise.all([...inFlight]);
+    }
+  };
+
+  const attached = [];
+
+  // CDP is what makes a download a *person* started observable, and it is the
+  // only source Puppeteer has, so it is tried first. It is also exclusive:
+  // `Browser.setDownloadBehavior` takes the bytes away from Playwright's own
+  // download handling, so attaching both sources would mean one download
+  // reported twice and a Playwright `path()` pointing at a file that was
+  // never written.
+  let cdpSession;
+  try {
+    cdpSession = await openBrowserCdpSession({ engine, browser, page });
+    if (cdpSession) {
+      attached.push(await attachCdpSource({ session: cdpSession, root, sink }));
+    }
+  } catch (error) {
+    cdpSession = undefined;
+    log?.warn?.(`manual downloads are not observable: ${error.message}`);
+  }
+
+  // Without Browser-domain access - an attached browser that refuses it, or a
+  // context with no page to borrow a session from - Playwright's own event
+  // still covers every automated download.
+  if (attached.length === 0 && engine === 'playwright' && context?.on) {
+    attached.push(attachPlaywrightSource({ context, sink }));
+  }
+
+  if (attached.length === 0) {
+    log?.warn?.(
+      'no download source could be attached: downloads will not be managed'
+    );
+  }
+
+  /**
+   * Wait for a download, optionally triggering it first.
+   *
+   * @param {Object} [captureOptions] - Capture options
+   * @param {Function} [captureOptions.action] - Action that triggers the download
+   * @param {Function|string} [captureOptions.filename] - Name for this download
+   * @param {number} [captureOptions.timeout] - Budget for the whole capture
+   * @param {Function} [captureOptions.validate] - Validation for this download
+   * @returns {Promise<Object>} The settled artifact
+   */
+  const capture = async (captureOptions = {}) => {
+    const {
+      action,
+      filename: captureFilename,
+      timeout = DEFAULT_CAPTURE_TIMEOUT,
+      validate: captureValidate,
+    } = captureOptions;
+
+    const deadline = createDeadline({ timeout });
+    const armedAt = artifacts.length;
+
+    let settle;
+    const settled = new Promise((resolve) => {
+      settle = resolve;
+    });
+
+    const waiter = {
+      /**
+       * Accept the first download that started after this waiter was armed.
+       *
+       * @param {Object} artifact - Settled artifact
+       * @returns {boolean} Whether this waiter claims it
+       */
+      accepts: (artifact) => artifacts.indexOf(artifact) >= armedAt,
+      settle,
+    };
+    waiters.add(waiter);
+
+    // Per-download naming and validation are attached to whichever download
+    // this waiter claims, so they cannot leak onto an unrelated one.
+    const claim = (artifact) => {
+      if (captureFilename !== undefined) {
+        artifact.filename =
+          typeof captureFilename === 'function'
+            ? captureFilename
+            : () => captureFilename;
+      }
+      if (captureValidate) {
+        artifact.validate = captureValidate;
+      }
+    };
+
+    const onStarted = (artifact) => {
+      if (waiter.accepts(artifact)) {
+        claim(artifact);
+      }
+    };
+    emitter.on(DOWNLOAD_EVENT.STARTED, onStarted);
+
+    // Both the waiter and the naming rules are armed before the action runs.
+    // A download can start, finish and be saved inside `action()` - attaching
+    // them afterwards would save the file under the wrong name and skip the
+    // caller's validation entirely.
+    try {
+      if (action) {
+        await action();
+      }
+    } catch (error) {
+      waiters.delete(waiter);
+      emitter.off(DOWNLOAD_EVENT.STARTED, onStarted);
+      throw error;
+    }
+
+    let timer;
+    const expiry = new Promise((_resolve, reject) => {
+      timer = setTimeout(
+        () =>
+          reject(
+            new Error(
+              `no download completed within ${timeout}ms of the triggering action`
+            )
+          ),
+        Math.max(1, deadline.remainingMs())
+      );
+    });
+
+    try {
+      const artifact = await Promise.race([settled, expiry]);
+      if (artifact.state !== DOWNLOAD_EVENT.COMPLETED) {
+        throw new Error(
+          `download ${artifact.id} ${artifact.state}: ${artifact.failure}`
+        );
+      }
+      return artifact;
+    } finally {
+      clearTimeout(timer);
+      waiters.delete(waiter);
+      emitter.off(DOWNLOAD_EVENT.STARTED, onStarted);
+    }
+  };
+
+  return {
+    directory: root,
+    persist,
+    conflict,
+    /**
+     * List every download seen in this session.
+     *
+     * @returns {Object[]} Artifacts in the order they started
+     */
+    list: () => [...artifacts],
+    capture,
+    idle,
+    on: (event, listener) => emitter.on(event, listener),
+    once: (event, listener) => emitter.once(event, listener),
+    off: (event, listener) => emitter.off(event, listener),
+    /**
+     * Stop observing downloads.
+     *
+     * Saved files are untouched: persistence is the point of the manager, so
+     * detaching must never be the thing that removes a user's file.
+     *
+     * @returns {Promise<void>}
+     */
+    dispose: async () => {
+      // A download still being written is finished first: detaching must not
+      // be the reason a file only ever exists under its `.partial` name.
+      await idle();
+      for (const handle of attached) {
+        await handle.detach();
+      }
+      await cdpSession?.detach?.().catch?.(() => {});
+      emitter.removeAllListeners();
+    },
+  };
+}
+
+export { classifyFailure, DOWNLOAD_CONFLICT, DOWNLOAD_FAILURE };

package/src/downloads/naming.js

@@ -0,0 +1,246 @@
+/**
+ * Naming and placement rules for managed downloads (issue #88).
+ *
+ * Everything here answers one question: given a name a *page* chose, where is
+ * it safe to write the bytes? A suggested filename is attacker-controlled
+ * input, so it is treated as a hint and never as a path.
+ */
+
+import path from 'node:path';
+
+/** The type a server sends when it cannot name the format either. */
+const GENERIC_BINARY_TYPE = 'application/octet-stream';
+
+/** Extensions derived from a declared or detected MIME type. */
+const MIME_EXTENSIONS = new Map([
+  ['application/pdf', '.pdf'],
+  ['application/json', '.json'],
+  ['application/zip', '.zip'],
+  ['application/gzip', '.gz'],
+  ['application/x-tar', '.tar'],
+  [GENERIC_BINARY_TYPE, '.bin'],
+  ['application/vnd.ms-excel', '.xls'],
+  [
+    'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+    '.xlsx',
+  ],
+  ['application/msword', '.doc'],
+  [
+    'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+    '.docx',
+  ],
+  ['text/csv', '.csv'],
+  ['text/html', '.html'],
+  ['text/plain', '.txt'],
+  ['image/png', '.png'],
+  ['image/jpeg', '.jpg'],
+  ['image/gif', '.gif'],
+  ['image/svg+xml', '.svg'],
+  ['image/webp', '.webp'],
+  ['video/mp4', '.mp4'],
+  ['audio/mpeg', '.mp3'],
+]);
+
+/** Leading bytes that identify a format regardless of what the page claimed. */
+const MAGIC_NUMBERS = [
+  { extension: '.pdf', bytes: [0x25, 0x50, 0x44, 0x46] },
+  { extension: '.png', bytes: [0x89, 0x50, 0x4e, 0x47] },
+  { extension: '.gif', bytes: [0x47, 0x49, 0x46, 0x38] },
+  { extension: '.jpg', bytes: [0xff, 0xd8, 0xff] },
+  { extension: '.zip', bytes: [0x50, 0x4b, 0x03, 0x04] },
+  { extension: '.gz', bytes: [0x1f, 0x8b] },
+];
+
+/** Name used when a page suggests nothing usable at all. */
+const FALLBACK_NAME = 'download';
+
+/** Windows device names that cannot be used as files even on other systems. */
+const RESERVED_NAMES = new Set([
+  'con',
+  'prn',
+  'aux',
+  'nul',
+  ...Array.from({ length: 9 }, (_, index) => `com${index + 1}`),
+  ...Array.from({ length: 9 }, (_, index) => `lpt${index + 1}`),
+]);
+
+/** A name that is only dots carries no information and is not a name. */
+const ONLY_DOTS = /^\.+$/;
+
+/** One whitespace character, tested one position at a time. */
+const WHITESPACE = /\s/;
+
+/**
+ * Trim the padding off a suggested name: leading whitespace, and trailing
+ * whitespace or dots.
+ *
+ * `/^\s+|[\s.]+$/` says the same thing in one line, but it backtracks
+ * quadratically over a long run of whitespace, and this string was chosen by
+ * the page. Walking in from both ends is one pass and cannot be made to cost
+ * more. Leading dots survive on purpose: `.bashrc` is a name, not padding.
+ *
+ * @param {string} value - Name to trim
+ * @returns {string} The name without its padding
+ */
+function trimNameEdges(value) {
+  let start = 0;
+  let end = value.length;
+
+  while (start < end && WHITESPACE.test(value[start])) {
+    start += 1;
+  }
+  while (
+    end > start &&
+    (value[end - 1] === '.' || WHITESPACE.test(value[end - 1]))
+  ) {
+    end -= 1;
+  }
+
+  return value.slice(start, end);
+}
+
+/**
+ * Strip a page-supplied name down to something that can only ever be a file
+ * inside the download root.
+ *
+ * Path separators, drive letters, control characters and `..` segments are
+ * removed rather than rejected, because a rejected download is a lost download
+ * and the caller asked for the file, not for the page's spelling of it.
+ *
+ * @param {string} suggested - Name suggested by the page or engine
+ * @returns {string} A single safe path segment
+ */
+export function sanitizeDownloadName(suggested) {
+  const raw = typeof suggested === 'string' ? suggested : '';
+
+  // Take the last segment under both separators: a page may suggest
+  // `../../etc/passwd` or `C:\Windows\system32\x`, and neither is a location
+  // we are willing to honor.
+  const lastSegment = raw.split(/[/\\]/).pop() ?? '';
+
+  const cleaned = trimNameEdges(
+    lastSegment
+      // eslint-disable-next-line no-control-regex -- control characters in a filename are exactly what we are removing
+      .replace(/[\u0000-\u001f\u007f]/g, '')
+      .replace(/[:*?"<>|]/g, '_')
+  );
+
+  if (!cleaned || ONLY_DOTS.test(cleaned)) {
+    return FALLBACK_NAME;
+  }
+
+  const stem = cleaned.slice(0, cleaned.lastIndexOf('.') + 1 || undefined);
+  if (RESERVED_NAMES.has(stem.replace(/\.$/, '').toLowerCase())) {
+    return `_${cleaned}`;
+  }
+
+  return cleaned;
+}
+
+/**
+ * Guess an extension from the first bytes of the file.
+ *
+ * @param {Buffer|Uint8Array|undefined} head - Leading bytes of the download
+ * @returns {string} Extension including the dot, or an empty string
+ */
+export function extensionFromContent(head) {
+  if (!head || head.length === 0) {
+    return '';
+  }
+
+  const found = MAGIC_NUMBERS.find(({ bytes }) =>
+    bytes.every((byte, index) => head[index] === byte)
+  );
+  return found ? found.extension : '';
+}
+
+/**
+ * Give a name an extension when the page did not supply one.
+ *
+ * Pages hand out UUID-like names constantly; a file called
+ * `7f1c...-9ab2` is unusable to a human and unopenable by the OS, so the
+ * declared MIME type (or the bytes themselves) supplies the missing suffix.
+ *
+ * @param {Object} options - Naming options
+ * @param {string} options.name - Sanitized name
+ * @param {string} [options.mimeType] - MIME type declared by the server
+ * @param {Buffer|Uint8Array} [options.head] - Leading bytes of the file
+ * @returns {string} Name with an extension when one could be determined
+ */
+export function withExtension({ name, mimeType, head }) {
+  if (path.extname(name)) {
+    return name;
+  }
+
+  const declaredType = String(mimeType ?? '')
+    .split(';')[0]
+    .trim()
+    .toLowerCase();
+  const declared = MIME_EXTENSIONS.get(declaredType);
+  const sniffed = extensionFromContent(head);
+  // `application/octet-stream` is what a server sends when it does not know
+  // either, so the bytes outrank it; any other declared type is a real claim.
+  const extension =
+    declaredType === GENERIC_BINARY_TYPE
+      ? sniffed || declared
+      : declared || sniffed;
+  return extension ? `${name}${extension}` : name;
+}
+
+/**
+ * Report whether a resolved path stays inside the download root.
+ *
+ * The check is done on resolved paths rather than on the name, so a symlinked
+ * or relative root cannot be used to step outside it.
+ *
+ * @param {string} root - Absolute download directory
+ * @param {string} candidate - Absolute candidate path
+ * @returns {boolean} Whether the candidate is inside the root
+ */
+export function isInsideRoot(root, candidate) {
+  const relative = path.relative(path.resolve(root), path.resolve(candidate));
+  return (
+    relative !== '' &&
+    !relative.startsWith('..') &&
+    !path.isAbsolute(relative) &&
+    !relative.split(path.sep).includes('..')
+  );
+}
+
+/**
+ * Resolve a safe absolute path for a download inside the root.
+ *
+ * @param {string} root - Absolute download directory
+ * @param {string} name - Sanitized file name
+ * @returns {string} Absolute path inside the root
+ * @throws {Error} When the name would escape the root
+ */
+export function resolveInsideRoot(root, name) {
+  const candidate = path.resolve(root, name);
+  if (!isInsideRoot(root, candidate)) {
+    throw new Error(
+      `refusing to write "${name}" outside the download directory ${root}`
+    );
+  }
+  return candidate;
+}
+
+/**
+ * Produce the candidate names a rename-on-conflict policy tries, in order.
+ *
+ * `report.pdf`, `report (2).pdf`, `report (3).pdf`, … — the order is fixed, so
+ * two runs of the same scenario produce the same names.
+ *
+ * @param {string} name - Sanitized file name
+ * @param {number} attempt - Zero-based attempt number
+ * @returns {string} Candidate name for that attempt
+ */
+export function renamedCandidate(name, attempt) {
+  if (attempt === 0) {
+    return name;
+  }
+
+  const extension = path.extname(name);
+  const stem = extension ? name.slice(0, -extension.length) : name;
+  return `${stem} (${attempt + 1})${extension}`;
+}