browser-debugger-cli 0.15.0 → 0.16.0

package/dist/telemetry/downloads.js

@@ -0,0 +1,265 @@
+/**
+ * Download tracking: where a session's downloads go, and what became of them.
+ */
+import * as fs from 'fs';
+import * as path from 'path';
+import { createLogger } from '../ui/logging/index.js';
+import { downloadsNotRedirectedWarning } from '../ui/messages/commands.js';
+import { getErrorMessage } from '../utils/errors.js';
+const log = createLogger('downloads');
+/** Name given to a download whose suggested name is not a usable file name */
+const FALLBACK_FILE_NAME = 'download';
+/**
+ * Track the session's downloads.
+ *
+ * In a directory, Chrome saves each download under its id (`allowAndName`),
+ * and the file is renamed to the suggested name once complete; the name is
+ * chosen when the download begins (`report (1).txt` when `report.txt` exists
+ * or was chosen for another), so a download still running already reports
+ * where it will be. In the browser's place, its own download settings stay
+ * and only its events are enabled. Refused downloads are canceled by Chrome
+ * and recorded with the reason.
+ *
+ * Chrome keeps a download behavior only while the connection that set it is
+ * open, so {@link attach} applies it again on another connection when the
+ * first one is lost. A browser-level connection also receives download
+ * events of other tabs (`target=_blank` links, `window.open()`), which do not
+ * reach a page's connection.
+ */
+export class DownloadTracker {
+    record;
+    destination;
+    reserved = new Set();
+    applied = { kind: 'browser' };
+    unsubscribe = () => undefined;
+    /** Set once stopped: no connection is followed, nor any behavior set, afterwards */
+    stopped = false;
+    /** Number of the latest attach: an earlier one still running gives way to it */
+    latestAttach = 0;
+    /** Attaches run one after another */
+    attaching = Promise.resolve();
+    /**
+     * @param record - Session record receiving downloads and the warning
+     * @param destination - Where downloads should go
+     */
+    constructor(record, destination) {
+        this.record = record;
+        this.destination = destination;
+        if (destination.kind === 'refused')
+            record.downloadsWarning = destination.reason;
+    }
+    /**
+     * Apply the destination on a connection and follow its download events
+     * there (instead of on the previous one). When Chrome refuses it, reports
+     * stop claiming bdg's directory (downloads go where the browser puts them)
+     * and the record carries a warning.
+     *
+     * Attaches run one at a time, and the latest wins: one called meanwhile
+     * (the browser-level connection lost while it was being set up) makes an
+     * earlier one give way without following its connection or warning. After
+     * {@link stop}, nothing is followed and no behavior is set.
+     *
+     * @param cdp - Connection (browser-level when possible)
+     * @returns True when Chrome took the destination on this connection
+     */
+    attach(cdp) {
+        const attempt = ++this.latestAttach;
+        const run = this.attaching.then(() => this.applyOn(cdp, attempt));
+        this.attaching = run.catch(() => undefined);
+        return run;
+    }
+    /** Stop following download events */
+    stop() {
+        this.stopped = true;
+        this.unsubscribe();
+    }
+    /**
+     * Whether an attach was superseded by a later one or by {@link stop}.
+     *
+     * @param attempt - Number of the attach
+     * @returns True when it must give way
+     */
+    superseded(attempt) {
+        return this.stopped || attempt !== this.latestAttach;
+    }
+    /**
+     * Set the behavior on a connection and follow its events, unless the
+     * attach was superseded before or while Chrome answered.
+     *
+     * @param cdp - Connection
+     * @param attempt - Number of the attach
+     * @returns True when Chrome took the destination and the attach still stands
+     */
+    async applyOn(cdp, attempt) {
+        if (this.superseded(attempt))
+            return false;
+        this.unsubscribe();
+        const error = await setDownloadBehavior(cdp, this.destination);
+        if (this.superseded(attempt))
+            return false;
+        this.applied = error === undefined ? this.destination : { kind: 'browser' };
+        if (this.destination.kind === 'directory') {
+            this.record.downloadsWarning =
+                error === undefined ? undefined : downloadsNotRedirectedWarning(error);
+        }
+        const handlers = [
+            cdp.on('Browser.downloadWillBegin', (event) => this.begin(event)),
+            cdp.on('Browser.downloadProgress', (progress) => {
+                const download = this.record.downloads.findLast((entry) => entry.guid === progress.guid);
+                if (download?.state !== 'inProgress')
+                    return;
+                updateDownload(download, progress, this.applied, this.reserved);
+            }),
+        ];
+        this.unsubscribe = () => handlers.forEach((remove) => remove());
+        return error === undefined;
+    }
+    /**
+     * Record a download that began, with the path chosen for it in bdg's directory.
+     *
+     * @param event - `Browser.downloadWillBegin` parameters
+     */
+    begin({ guid, url, suggestedFilename }) {
+        const dir = this.applied.kind === 'directory' ? this.applied.dir : undefined;
+        const target = dir && reserveDownloadPath(dir, suggestedFilename, this.reserved);
+        this.record.downloads.push({
+            guid,
+            url,
+            suggestedFilename,
+            state: 'inProgress',
+            ...(target && { path: target }),
+        });
+    }
+}
+/**
+ * A tracked download as commands report it, as it is now.
+ *
+ * @param download - Tracked download
+ * @returns Copy without Chrome's id
+ */
+export function toDownloadInfo(download) {
+    const { url, suggestedFilename, path: file, state, bytes, reason } = download;
+    return {
+        url,
+        suggestedFilename,
+        ...(file !== undefined && { path: file }),
+        state,
+        ...(bytes !== undefined && { bytes }),
+        ...(reason !== undefined && { reason }),
+    };
+}
+/**
+ * Set the browser's download behavior for the destination and enable
+ * download events.
+ *
+ * @param cdp - CDP connection
+ * @param destination - Where downloads go
+ * @returns Why Chrome refused it, or undefined when it took it
+ */
+async function setDownloadBehavior(cdp, destination) {
+    const behavior = destination.kind === 'directory'
+        ? { behavior: 'allowAndName', downloadPath: destination.dir }
+        : { behavior: destination.kind === 'refused' ? 'deny' : 'default' };
+    try {
+        await cdp.send('Browser.setDownloadBehavior', { ...behavior, eventsEnabled: true });
+        return undefined;
+    }
+    catch (error) {
+        log.info(`Download behavior not set: ${getErrorMessage(error)}`);
+        return getErrorMessage(error);
+    }
+}
+/**
+ * Apply a progress event: bytes so far, then the final state; a completed
+ * download in bdg's directory is renamed from its id to its chosen name.
+ *
+ * @param download - Download being updated
+ * @param progress - Progress event
+ * @param destination - Where downloads go
+ * @param reserved - Paths chosen for downloads still running
+ */
+function updateDownload(download, progress, destination, reserved) {
+    download.bytes = progress.receivedBytes;
+    if (progress.state === 'inProgress')
+        return;
+    if (download.path)
+        reserved.delete(reservationKey(download.path));
+    if (progress.state === 'canceled') {
+        delete download.path;
+        if (destination.kind === 'refused')
+            download.reason = destination.reason;
+    }
+    else if (destination.kind === 'directory') {
+        download.path = saveUnderChosenName(destination.dir, download, reserved);
+    }
+    else if (progress.filePath) {
+        download.path = progress.filePath;
+    }
+    download.state = progress.state;
+}
+/**
+ * Rename a completed download from its id to the name chosen for it (another
+ * one when a file took that name meanwhile).
+ *
+ * @param downloadDir - Directory downloads are saved into
+ * @param download - Completed download
+ * @param reserved - Paths chosen for downloads still running
+ * @returns Path of the file: the chosen one, or its id's when renaming failed
+ */
+function saveUnderChosenName(downloadDir, download, reserved) {
+    const saved = path.join(downloadDir, download.guid);
+    const chosen = download.path && !fs.existsSync(download.path)
+        ? download.path
+        : reserveDownloadPath(downloadDir, download.suggestedFilename, reserved);
+    reserved.delete(reservationKey(chosen));
+    try {
+        fs.renameSync(saved, chosen);
+        return chosen;
+    }
+    catch (error) {
+        log.debug(`Download ${download.guid} kept under its id: ${getErrorMessage(error)}`);
+        return saved;
+    }
+}
+/**
+ * Key of a reserved path: lowercased, so `Report.txt` and `report.txt` do not
+ * both get chosen on a case-insensitive file system (macOS, Windows).
+ *
+ * @param file - Path
+ * @returns Key
+ */
+function reservationKey(file) {
+    return file.toLowerCase();
+}
+/**
+ * Choose a free path for a download: its suggested name, or with ` (1)`,
+ * ` (2)`… before the extension when a file or another running download has
+ * that name.
+ *
+ * @param downloadDir - Directory downloads are saved into
+ * @param suggestedFilename - Name the page or server suggested
+ * @param reserved - Paths chosen for downloads still running (the result is added)
+ * @returns Absolute path
+ */
+export function reserveDownloadPath(downloadDir, suggestedFilename, reserved) {
+    const name = safeFileName(suggestedFilename);
+    const { name: stem, ext } = path.parse(name);
+    for (let copy = 0;; copy++) {
+        const candidate = path.join(downloadDir, copy === 0 ? name : `${stem} (${copy})${ext}`);
+        if (reserved.has(reservationKey(candidate)) || fs.existsSync(candidate))
+            continue;
+        reserved.add(reservationKey(candidate));
+        return candidate;
+    }
+}
+/**
+ * A suggested file name reduced to a name in the download directory.
+ *
+ * @param suggestedFilename - Name the page or server suggested
+ * @returns Its last path segment, or {@link FALLBACK_FILE_NAME} when that is empty, `.` or `..`
+ */
+function safeFileName(suggestedFilename) {
+    const name = path.basename(suggestedFilename.replaceAll('\\', '/'));
+    return name === '' || name === '.' || name === '..' ? FALLBACK_FILE_NAME : name;
+}
+//# sourceMappingURL=downloads.js.map

package/dist/telemetry/har/builder.js

@@ -101,6 +101,7 @@ function buildWebSocketMessage(frame) {
         time: frame.timestamp / 1000,
         opcode: frame.opcode,
         data: frame.payloadData,
+        ...(frame.truncatedFrom !== undefined && { _truncatedFrom: frame.truncatedFrom }),
     };
 }
 /**
@@ -119,7 +120,7 @@ function buildRequest(req) {
         headers: convertHeaders(req.requestHeaders),
         queryString: extractQueryParams(url),
         headersSize: estimateRequestHeadersSize(req.method, req.url, req.requestHeaders),
-        bodySize: req.requestBody ? Buffer.byteLength(req.requestBody, 'utf-8') : 0,
+        bodySize: requestBodySize(req),
     };
     const postData = buildPostData(req);
     if (postData) {
@@ -189,7 +190,21 @@ function contentLength(req) {
     return Number.isInteger(bytes) && bytes >= 0 ? bytes : undefined;
 }
 /**
- * Build POST data object if request has body.
+ * Size of the request body: unknown (-1) when it was evicted at the body budget.
+ *
+ * @param req - Network request data
+ * @returns Body size in bytes, or -1
+ */
+function requestBodySize(req) {
+    if (!req.requestBody)
+        return 0;
+    if (skippedBodyReason(req.requestBody) !== undefined)
+        return -1;
+    return Buffer.byteLength(req.requestBody, 'utf-8');
+}
+/**
+ * Build POST data object if request has body. A body bdg did not keep is
+ * exported without `text` and with the reason as `comment`.
  *
  * @param req - Network request data
  * @returns POST data object or undefined
@@ -197,11 +212,11 @@ function contentLength(req) {
 function buildPostData(req) {
     if (!req.requestBody)
         return undefined;
-    const contentType = getHeader(req.requestHeaders, 'content-type') ?? 'text/plain';
-    return {
-        mimeType: contentType,
-        text: req.requestBody,
-    };
+    const mimeType = getHeader(req.requestHeaders, 'content-type') ?? 'text/plain';
+    const skipped = skippedBodyReason(req.requestBody);
+    if (skipped !== undefined)
+        return { mimeType, comment: `Body not captured: ${skipped}` };
+    return { mimeType, text: req.requestBody };
 }
 /**
  * Duration between two CDP timing offsets (ms), or -1 if either is unknown.

package/dist/telemetry/har/sanitize.d.ts

@@ -9,9 +9,13 @@
  * `httpOnly`) with the value `[redacted]`, so the export still shows that a
  * request was authenticated and which cookies were set. It also covers API
  * key, token and session headers, credential query parameters in URLs
- * (`?code=`, `?access_token=`) and credential fields of request bodies
- * (sanitizeBody.ts). `headersSize` and `bodySize` stay those of the captured
- * request. Response bodies and WebSocket messages are not redacted.
+ * (`?code=`, `?access_token=`), and credential fields of request and response
+ * bodies and of WebSocket text messages (sanitizeBody.ts), editing only
+ * those values. `headersSize`, `bodySize` and `content.size` stay those of the
+ * captured request. Base64 bodies are decoded when their type is generic,
+ * JSON, form or event-stream, and binary WebSocket messages always; those
+ * that are not UTF-8 text, and other binary bodies, are kept. A body or
+ * message the sanitizer fails on is replaced whole by {@link REDACTED}.
  */
 import type { Entry } from './types.js';
 /**

package/dist/telemetry/har/sanitize.js

@@ -9,11 +9,15 @@
  * `httpOnly`) with the value `[redacted]`, so the export still shows that a
  * request was authenticated and which cookies were set. It also covers API
  * key, token and session headers, credential query parameters in URLs
- * (`?code=`, `?access_token=`) and credential fields of request bodies
- * (sanitizeBody.ts). `headersSize` and `bodySize` stay those of the captured
- * request. Response bodies and WebSocket messages are not redacted.
+ * (`?code=`, `?access_token=`), and credential fields of request and response
+ * bodies and of WebSocket text messages (sanitizeBody.ts), editing only
+ * those values. `headersSize`, `bodySize` and `content.size` stay those of the
+ * captured request. Base64 bodies are decoded when their type is generic,
+ * JSON, form or event-stream, and binary WebSocket messages always; those
+ * that are not UTF-8 text, and other binary bodies, are kept. A body or
+ * message the sanitizer fails on is replaced whole by {@link REDACTED}.
  */
-import { REDACTED, isSensitiveField, redactPairs, redactRequestBody, } from './sanitizeBody.js';
+import { REDACTED, isSensitiveField, redactBase64Body, redactBody, redactOrReplaceWhole, redactPairs, } from './sanitizeBody.js';
 /** {@link REDACTED} as written in a URL */
 const URL_REDACTED = encodeURIComponent(REDACTED);
 /** Headers whose values are credentials, lowercased (others match by pattern) */
@@ -36,6 +40,12 @@ const SENSITIVE_HEADER_SEGMENT = /(^|-)(api-?key|apikey|token|secret|jwt|subscri
 const URL_HEADERS = new Set(['location', 'referer']);
 /** Query parameter names that hold credentials in URLs only (OAuth codes, signed URLs, API keys) */
 const SENSITIVE_URL_PARAM = /^(code|sig|key)$/i;
+/** {@link REDACTED} as base64, for a binary body or message replaced whole */
+const BASE64_REDACTED = Buffer.from(REDACTED).toString('base64');
+/** WebSocket opcode of a text message */
+const TEXT_OPCODE = 1;
+/** WebSocket opcode of a binary message (base64 in the HAR) */
+const BINARY_OPCODE = 2;
 /**
  * Redact the credentials of a HAR entry.
  *
@@ -43,7 +53,7 @@ const SENSITIVE_URL_PARAM = /^(code|sig|key)$/i;
  * @returns Copy of the entry with credential values replaced by {@link REDACTED}
  */
 export function sanitizeEntry(entry) {
-    const { request, response } = entry;
+    const { request, response, _webSocketMessages: messages } = entry;
     const postData = request.postData;
     return {
         ...entry,
@@ -54,7 +64,10 @@ export function sanitizeEntry(entry) {
             headers: request.headers.map(redactHeader),
             queryString: request.queryString.map(redactQueryParam),
             ...(postData?.text !== undefined && {
-                postData: { ...postData, text: redactRequestBody(postData.text, postData.mimeType) },
+                postData: {
+                    ...postData,
+                    text: redactOrReplaceWhole(postData.text, (text) => redactBody(text, postData.mimeType)),
+                },
             }),
         },
         response: {
@@ -62,9 +75,42 @@ export function sanitizeEntry(entry) {
             cookies: response.cookies.map(redactCookie),
             headers: response.headers.map(redactHeader),
             redirectURL: redactUrl(response.redirectURL),
+            content: redactContent(response.content),
         },
+        ...(messages && { _webSocketMessages: messages.map(redactWebSocketMessage) }),
     };
 }
+/**
+ * Redact credential fields of a response body, also of a base64 body whose
+ * type is generic or JSON.
+ *
+ * @param content - HAR response content
+ * @returns The content, or a copy with its text redacted; `size` stays as captured
+ */
+function redactContent(content) {
+    const { text, mimeType } = content;
+    if (text === undefined)
+        return content;
+    const redacted = content.encoding === 'base64'
+        ? redactOrReplaceWhole(text, (body) => redactBase64Body(body, mimeType), BASE64_REDACTED)
+        : redactOrReplaceWhole(text, (body) => redactBody(body, mimeType));
+    return redacted === text ? content : { ...content, text: redacted };
+}
+/**
+ * Redact credential fields of a WebSocket message: a text message, or a
+ * binary one that decodes as UTF-8 (redacted and encoded again).
+ *
+ * @param message - HAR WebSocket message
+ * @returns The message, or a copy with its data redacted
+ */
+function redactWebSocketMessage(message) {
+    if (message.opcode !== TEXT_OPCODE && message.opcode !== BINARY_OPCODE)
+        return message;
+    const data = message.opcode === TEXT_OPCODE
+        ? redactOrReplaceWhole(message.data, (text) => redactBody(text, ''))
+        : redactOrReplaceWhole(message.data, (text) => redactBase64Body(text, ''), BASE64_REDACTED);
+    return data === message.data ? message : { ...message, data };
+}
 /**
  * Whether a header carries a credential.
  *

package/dist/telemetry/har/sanitizeBody.d.ts

@@ -1,9 +1,18 @@
 /**
- * Credential redaction in request bodies and `name=value` lists, for
- * sanitized HAR exports (see sanitize.ts).
+ * Credential redaction in request and response bodies, WebSocket text
+ * messages and `name=value` lists, for sanitized HAR exports (see sanitize.ts).
  *
  * Matching is by field name only, so it over-redacts: any primitive whose
  * name looks like a credential (`tokenCount: 5`) is replaced too.
+ *
+ * JSON is never parsed and re-serialized: a single linear scan replaces the
+ * credential values in place, so everything else (64-bit numbers, formatting,
+ * duplicate keys, a BOM or `)]}'` prefix) stays byte for byte, and truncated
+ * JSON, socket.io and SockJS packets, server-sent events, NDJSON and JSON
+ * encoded in string values are covered too. JWTs are redacted under any name.
+ *
+ * Only JSON syntax is understood: single-quoted strings, unquoted keys,
+ * JSONP and `name:value` header lines (STOMP `passcode:`) are not.
  */
 /** Replaces a credential value */
 export declare const REDACTED = "[redacted]";
@@ -15,6 +24,26 @@ export declare const REDACTED = "[redacted]";
  * @returns True for password, token, secret, key, session and signature names
  */
 export declare function isSensitiveField(name: string): boolean;
+/**
+ * Run a redaction, replacing the whole text when it fails, so that a body
+ * the sanitizer cannot handle is never exported as captured.
+ *
+ * @param text - Body or message text
+ * @param redact - Redaction of the text
+ * @param whole - Text written when the redaction throws
+ * @returns Redacted text, or `whole`
+ */
+export declare function redactOrReplaceWhole(text: string, redact: (text: string) => string, whole?: string): string;
+/**
+ * Replace every whole JWT (`eyJ` and three base64url segments, not part of a
+ * longer word) in text. Scanned by hand: a regular expression overflows the
+ * stack on a word of millions of characters.
+ *
+ * @param text - Text
+ * @param replacement - Text written instead of each JWT
+ * @returns Text with JWTs replaced; the text unchanged when it has none
+ */
+export declare function redactJwts(text: string, replacement: string): string;
 /**
  * Redact credential values in an `&`-separated `name=value` list (a form body,
  * query string or fragment), leaving the other pairs byte for byte.
@@ -26,13 +55,24 @@ export declare function isSensitiveField(name: string): boolean;
  */
 export declare function redactPairs(text: string, isSensitive: (name: string) => boolean, replacement: string): string;
 /**
- * Redact credential fields of a request body: multipart parts, form fields
- * (by Content-Type or shape) and JSON fields at any depth.
+ * Redact credential fields of a body or WebSocket text message: multipart
+ * parts, form fields (by Content-Type or shape) and JSON fields at any depth.
  *
  * @param text - Body text
- * @param mimeType - Content-Type of the body
+ * @param mimeType - Content-Type of the body (empty for a WebSocket message)
  * @returns Body with credential values replaced; the text unchanged when
- *   there were none
+ *   there were none or it is neither JSON, a form nor multipart
+ */
+export declare function redactBody(text: string, mimeType: string): string;
+/**
+ * Redact credential fields of a base64 body or binary WebSocket message that
+ * may be text: one with no, a generic binary, a JSON, a form or an
+ * event-stream Content-Type that decodes as UTF-8.
+ *
+ * @param base64 - Body as base64
+ * @param mimeType - Content-Type of the body
+ * @returns The redacted body re-encoded, or the input unchanged when it was
+ *   not decodable text or held no credentials
  */
-export declare function redactRequestBody(text: string, mimeType: string): string;
+export declare function redactBase64Body(base64: string, mimeType: string): string;
 //# sourceMappingURL=sanitizeBody.d.ts.map