@d-zero/beholder 3.1.6 → 4.1.0

package/dist/scraper.js

@@ -43,11 +43,16 @@ import { retry as retryable } from '@d-zero/shared/retry';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
 import { resourceLog, scraperLog } from './debug.js';
 import { DEFAULT_DOM_EVALUATION_TIMEOUT, getAnchorList, getImageList, getMeta, } from './dom-evaluation.js';
+import { getMainContents } from './get-main-contents.js';
 import { isError } from './is-error.js';
+import { isHtmlContentType } from './is-html-content-type.js';
 import { keywordCheck } from './keyword-check.js';
+import { measureScrollHeight } from './measure-scroll-height.js';
 import { emptyMeta } from './meta/classify.js';
 import { findDisconnectionFailures } from './network-disconnection.js';
 import { parseUrl } from './parse-url.js';
+import { toConsoleLogEntry } from './to-console-log-entry.js';
+import { toPageErrorEntry } from './to-page-error-entry.js';
 const pid = `${process.pid}`;
 const log = scraperLog.extend(pid);
 const rLog = resourceLog.extend(pid);
@@ -95,7 +100,10 @@ let Scraper = (() => {
                 })];
             _private_fetchImages_decorators = [retryable({
                     timeout: 20 * 60 * 1000,
-                    fallback: [],
+                    fallback: {
+                        imageList: [],
+                        scrollHeight: { desktop: null, mobile: null },
+                    },
                     onWait(determinedInterval, retryCount, methodName, error) {
                         void this.emit('changePhase', {
                             pid: process.pid,
@@ -115,17 +123,21 @@ let Scraper = (() => {
                         });
                     },
                 })];
-            __esDecorate(this, _private_fetchData_descriptor = { value: __setFunctionName(async function (page, url, isExternal, captureImages, imageLoadTimeout, resources, failedRequests, options) {
+            __esDecorate(this, _private_fetchData_descriptor = { value: __setFunctionName(async function (page, url, isExternal, captureImages, imageLoadTimeout, resources, consoleLogs, failedRequests, options) {
                     const parseOpts = options?.disableQueries == null
                         ? undefined
                         : { disableQueries: options.disableQueries };
                     const domEvaluationTimeout = options?.domEvaluationTimeout ?? DEFAULT_DOM_EVALUATION_TIMEOUT;
                     const networkLogs = {};
+                    // Tracks in-flight `toConsoleLogEntry()` resolutions (async `jsonValue()`
+                    // extraction) so callers can await them before reading `consoleLogs`.
+                    const pendingConsoleWork = [];
                     // Clear stale state from previous retries (@retryable may re-invoke this method
                     // with the same page and mutable arrays, so we must reset to avoid accumulation)
                     this.#cleanupPageListeners();
                     failedRequests.length = 0;
                     resources.length = 0;
+                    consoleLogs.length = 0;
                     // Define named listeners so they can be individually removed on retry/cleanup
                     const onDialog = async (dialog) => {
                         log(`Appear ${dialog.type()} dialog: ${dialog.message()}`);
@@ -141,6 +153,8 @@ let Scraper = (() => {
                     let onRequest = null;
                     let onResponse = null;
                     let onRequestFailed = null;
+                    let onConsole = null;
+                    let onPageError = null;
                     if (!isExternal) {
                         onRequest = (request) => {
                             const url = parseUrl(request.url(), parseOpts);
@@ -215,9 +229,23 @@ let Scraper = (() => {
                             rLog('Request failed: %s (%s)', request.url(), errorText);
                             failedRequests.push({ url: request.url(), errorText });
                         };
+                        onConsole = (msg) => {
+                            pendingConsoleWork.push(toConsoleLogEntry(msg, url.withoutHash).then((entry) => {
+                                consoleLogs.push(entry);
+                            }, (error) => {
+                                // A single malformed console message must not fail the whole
+                                // scrape (e.g. mid-navigation execution-context teardown).
+                                log('Error(CONSOLE_LOG): %s', error);
+                            }));
+                        };
+                        onPageError = (error) => {
+                            consoleLogs.push(toPageErrorEntry(error, url.withoutHash));
+                        };
                         page.on('request', onRequest);
                         page.on('response', onResponse);
                         page.on('requestfailed', onRequestFailed);
+                        page.on('console', onConsole);
+                        page.on('pageerror', onPageError);
                     }
                     // Store cleanup function for retry/post-fetch removal
                     this.#pageListenerCleanup = () => {
@@ -228,93 +256,184 @@ let Scraper = (() => {
                             page.off('response', onResponse);
                         if (onRequestFailed)
                             page.off('requestfailed', onRequestFailed);
+                        if (onConsole)
+                            page.off('console', onConsole);
+                        if (onPageError)
+                            page.off('pageerror', onPageError);
                     };
                     const navigationTimeout = options?.navigationTimeout ?? 60_000;
-                    void this.emit('changePhase', {
-                        pid: process.pid,
-                        name: 'openPage',
-                        url,
-                        isExternal,
-                        message: `%countdown(${navigationTimeout},openPage_${url.withoutHash},s)%s`,
-                    });
-                    if (url.username && url.password) {
-                        await page.setExtraHTTPHeaders({
-                            Authorization: `Basic ${Buffer.from(`${url.username}:${url.password}`).toString('base64')}`,
+                    // The whole navigation/extraction sequence below is wrapped in `finally`
+                    // so that `pendingConsoleWork` is always awaited exactly once — on every
+                    // return AND on every throw (e.g. `Page.goto returned null`, network
+                    // disconnection). A per-return-statement `await` would miss thrown exits
+                    // and let a still-resolving `toConsoleLogEntry()` push into `consoleLogs`
+                    // after a `@retryable` retry has already cleared it for a new attempt.
+                    try {
+                        void this.emit('changePhase', {
+                            pid: process.pid,
+                            name: 'openPage',
+                            url,
+                            isExternal,
+                            message: `%countdown(${navigationTimeout},openPage_${url.withoutHash},s)%s`,
                         });
-                    }
-                    const res = await page.goto(url.withoutHashAndAuth, { timeout: navigationTimeout });
-                    if (!res) {
-                        throw new Error('The method Page.goto returned null');
-                    }
-                    const destUrl = parseUrl(page.url(), parseOpts);
-                    const redirectPaths = new Set();
-                    if (url.withoutHash !== destUrl.withoutHash) {
-                        const redirectChain = res
-                            .request()
-                            .redirectChain()
-                            .map((req) => req.url());
-                        for (const redirectPath of redirectChain) {
-                            redirectPaths.add(redirectPath);
+                        if (url.username && url.password) {
+                            await page.setExtraHTTPHeaders({
+                                Authorization: `Basic ${Buffer.from(`${url.username}:${url.password}`).toString('base64')}`,
+                            });
                         }
-                        redirectPaths.add(destUrl.withoutHash);
-                    }
-                    if (destUrl.hostname !== url.hostname) {
-                        isExternal = true;
-                    }
-                    const status = res.status();
-                    const statusText = res.statusText();
-                    const responseHeaders = res.headers();
-                    const contentType = responseHeaders['content-type']?.split(';')[0] || null;
-                    const _contentLength = Number.parseInt(responseHeaders['content-length'] ?? '');
-                    const contentLength = Number.isFinite(_contentLength) ? _contentLength : null;
-                    if (contentType !== 'text/html') {
-                        return {
+                        const res = await page.goto(url.withoutHashAndAuth, { timeout: navigationTimeout });
+                        if (!res) {
+                            throw new Error('The method Page.goto returned null');
+                        }
+                        const destUrl = parseUrl(page.url(), parseOpts);
+                        const redirectPaths = new Set();
+                        if (url.withoutHash !== destUrl.withoutHash) {
+                            const redirectChain = res
+                                .request()
+                                .redirectChain()
+                                .map((req) => req.url());
+                            for (const redirectPath of redirectChain) {
+                                redirectPaths.add(redirectPath);
+                            }
+                            redirectPaths.add(destUrl.withoutHash);
+                        }
+                        if (destUrl.hostname !== url.hostname) {
+                            isExternal = true;
+                        }
+                        const status = res.status();
+                        const statusText = res.statusText();
+                        const responseHeaders = res.headers();
+                        const contentType = responseHeaders['content-type']?.split(';')[0] || null;
+                        const _contentLength = Number.parseInt(responseHeaders['content-length'] ?? '');
+                        const contentLength = Number.isFinite(_contentLength) ? _contentLength : null;
+                        if (!isHtmlContentType(contentType)) {
+                            return {
+                                url,
+                                isTarget: false,
+                                isExternal,
+                                redirectPaths: [...redirectPaths],
+                                status,
+                                statusText,
+                                contentType,
+                                contentLength,
+                                responseHeaders,
+                                meta: emptyMeta(),
+                                imageList: [],
+                                anchorList: [],
+                                html: '',
+                                mainContents: null,
+                                scrollHeight: null,
+                                isSkipped: false,
+                            };
+                        }
+                        void this.emit('changePhase', {
+                            pid: process.pid,
+                            name: 'loadDOMContent',
                             url,
-                            isTarget: false,
                             isExternal,
-                            redirectPaths: [...redirectPaths],
-                            status,
-                            statusText,
-                            contentType,
-                            contentLength,
-                            responseHeaders,
-                            meta: emptyMeta(),
-                            imageList: [],
-                            anchorList: [],
-                            html: '',
-                            isSkipped: false,
-                        };
-                    }
-                    void this.emit('changePhase', {
-                        pid: process.pid,
-                        name: 'loadDOMContent',
-                        url,
-                        isExternal,
-                        message: '',
-                    });
-                    await page
-                        .waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 5000 })
-                        .catch(() => { });
-                    void this.emit('changePhase', {
-                        pid: process.pid,
-                        name: 'getHTML',
-                        url,
-                        isExternal,
-                        message: '',
-                    });
-                    const { title, html } = await page.evaluate(() => {
-                        /* global document */
-                        return {
-                            title: document.title,
-                            html: document.documentElement.outerHTML,
-                        };
-                    });
-                    if (isExternal) {
-                        const externalMeta = emptyMeta();
-                        externalMeta.title = title;
+                            message: '',
+                        });
+                        await page
+                            .waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 5000 })
+                            .catch(() => { });
+                        void this.emit('changePhase', {
+                            pid: process.pid,
+                            name: 'getHTML',
+                            url,
+                            isExternal,
+                            message: '',
+                        });
+                        const { title, html } = await page.evaluate(() => {
+                            /* global document */
+                            return {
+                                title: document.title,
+                                html: document.documentElement.outerHTML,
+                            };
+                        });
+                        if (isExternal) {
+                            const externalMeta = emptyMeta();
+                            externalMeta.title = title;
+                            return {
+                                url,
+                                isTarget: false,
+                                isExternal,
+                                redirectPaths: [...redirectPaths],
+                                status,
+                                statusText,
+                                contentType,
+                                contentLength,
+                                responseHeaders,
+                                meta: externalMeta,
+                                imageList: [],
+                                anchorList: [],
+                                html,
+                                mainContents: null,
+                                scrollHeight: null,
+                                isSkipped: false,
+                            };
+                        }
+                        void this.emit('changePhase', {
+                            pid: process.pid,
+                            name: 'waitNetworkIdle',
+                            url,
+                            isExternal,
+                            message: '',
+                        });
+                        await page
+                            .waitForNavigation({ waitUntil: 'networkidle0', timeout: 5000 })
+                            .catch(() => { });
+                        // Check for network disconnection errors in failed requests
+                        const disconnectionFailures = findDisconnectionFailures(failedRequests);
+                        if (disconnectionFailures.length > 0) {
+                            const errorSummary = disconnectionFailures
+                                .map((r) => `${r.url} (${r.errorText})`)
+                                .join(', ');
+                            throw new Error(`Network disconnection detected during page load: ${errorSummary}`);
+                        }
+                        const mainContents = await getMainContents(page, {
+                            mainContentSelector: options?.mainContentSelector,
+                        });
+                        void this.emit('changePhase', {
+                            pid: process.pid,
+                            name: 'getAnchors',
+                            url,
+                            isExternal,
+                            message: `%countdown(${domEvaluationTimeout},getAnchors_${url.withoutHash},s)%s`,
+                        });
+                        const anchorList = await getAnchorList(page, parseOpts, domEvaluationTimeout);
+                        void this.emit('changePhase', {
+                            pid: process.pid,
+                            name: 'getMeta',
+                            url,
+                            isExternal,
+                            message: `%countdown(${domEvaluationTimeout},getMeta_${url.withoutHash},s)%s`,
+                        });
+                        const meta = await getMeta(page, {
+                            url: url.withoutHashAndAuth,
+                            html,
+                            statusCode: status,
+                            headers: responseHeaders ?? undefined,
+                        }, domEvaluationTimeout);
+                        let imageList = [];
+                        let scrollHeight = null;
+                        if (captureImages) {
+                            void this.emit('changePhase', {
+                                pid: process.pid,
+                                name: 'extractImages',
+                                url,
+                                isExternal,
+                                message: `%countdown(${domEvaluationTimeout},extractImages_${url.withoutHash},s)%s`,
+                            });
+                            const fetched = await this.#fetchImages(page, url.withoutHashAndAuth, isExternal, imageLoadTimeout, domEvaluationTimeout);
+                            imageList = fetched.imageList;
+                            scrollHeight = fetched.scrollHeight;
+                        }
+                        else {
+                            scrollHeight = await measureScrollHeight(page);
+                        }
                         return {
                             url,
-                            isTarget: false,
+                            isTarget: true,
                             isExternal,
                             redirectPaths: [...redirectPaths],
                             status,
@@ -322,80 +441,18 @@ let Scraper = (() => {
                             contentType,
                             contentLength,
                             responseHeaders,
-                            meta: externalMeta,
-                            imageList: [],
-                            anchorList: [],
+                            meta,
+                            anchorList,
+                            imageList,
                             html,
+                            mainContents,
+                            scrollHeight,
                             isSkipped: false,
                         };
                     }
-                    void this.emit('changePhase', {
-                        pid: process.pid,
-                        name: 'waitNetworkIdle',
-                        url,
-                        isExternal,
-                        message: '',
-                    });
-                    await page
-                        .waitForNavigation({ waitUntil: 'networkidle0', timeout: 5000 })
-                        .catch(() => { });
-                    // Check for network disconnection errors in failed requests
-                    const disconnectionFailures = findDisconnectionFailures(failedRequests);
-                    if (disconnectionFailures.length > 0) {
-                        const errorSummary = disconnectionFailures
-                            .map((r) => `${r.url} (${r.errorText})`)
-                            .join(', ');
-                        throw new Error(`Network disconnection detected during page load: ${errorSummary}`);
+                    finally {
+                        await Promise.all(pendingConsoleWork);
                     }
-                    void this.emit('changePhase', {
-                        pid: process.pid,
-                        name: 'getAnchors',
-                        url,
-                        isExternal,
-                        message: `%countdown(${domEvaluationTimeout},getAnchors_${url.withoutHash},s)%s`,
-                    });
-                    const anchorList = await getAnchorList(page, parseOpts, domEvaluationTimeout);
-                    void this.emit('changePhase', {
-                        pid: process.pid,
-                        name: 'getMeta',
-                        url,
-                        isExternal,
-                        message: `%countdown(${domEvaluationTimeout},getMeta_${url.withoutHash},s)%s`,
-                    });
-                    const meta = await getMeta(page, {
-                        url: url.withoutHashAndAuth,
-                        html,
-                        statusCode: status,
-                        headers: responseHeaders ?? undefined,
-                    }, domEvaluationTimeout);
-                    const imageList = captureImages
-                        ? await (async () => {
-                            void this.emit('changePhase', {
-                                pid: process.pid,
-                                name: 'extractImages',
-                                url,
-                                isExternal,
-                                message: `%countdown(${domEvaluationTimeout},extractImages_${url.withoutHash},s)%s`,
-                            });
-                            return this.#fetchImages(page, url.withoutHashAndAuth, isExternal, imageLoadTimeout, domEvaluationTimeout);
-                        })()
-                        : [];
-                    return {
-                        url,
-                        isTarget: true,
-                        isExternal,
-                        redirectPaths: [...redirectPaths],
-                        status,
-                        statusText,
-                        contentType,
-                        contentLength,
-                        responseHeaders,
-                        meta,
-                        anchorList,
-                        imageList,
-                        html,
-                        isSkipped: false,
-                    };
                 }, "#fetchData") }, _private_fetchData_decorators, { kind: "method", name: "#fetchData", static: false, private: true, access: { has: obj => #fetchData in obj, get: obj => obj.#fetchData }, metadata: _metadata }, null, _instanceExtraInitializers);
             __esDecorate(this, _private_fetchImages_descriptor = { value: __setFunctionName(async function (page, url, isExternal, imageLoadTimeout, domEvaluationTimeout) {
                     const listener = this.#createPageScanListener(isExternal);
@@ -404,7 +461,9 @@ let Scraper = (() => {
                         { key: 'mobile-small', preset: devicePresets['mobile-small'] },
                     ];
                     const imageList = [];
+                    const scrollHeight = { desktop: null, mobile: null };
                     for (const { key, preset } of devices) {
+                        const scrollKey = key === 'desktop-compact' ? 'desktop' : 'mobile';
                         try {
                             void this.emit('changePhase', {
                                 pid: process.pid,
@@ -421,6 +480,7 @@ let Scraper = (() => {
                                 timeout: 5000,
                                 maxScrollHeight: MAX_SCROLL_HEIGHT,
                             });
+                            scrollHeight[scrollKey] = scanResult.scrollHeight;
                             if (!scanResult.scrolled) {
                                 void this.emit('changePhase', {
                                     pid: process.pid,
@@ -465,7 +525,7 @@ let Scraper = (() => {
                             });
                         }
                     }
-                    return imageList;
+                    return { imageList, scrollHeight };
                 }, "#fetchImages") }, _private_fetchImages_decorators, { kind: "method", name: "#fetchImages", static: false, private: true, access: { has: obj => #fetchImages in obj, get: obj => obj.#fetchImages }, metadata: _metadata }, null, _instanceExtraInitializers);
             if (_metadata) Object.defineProperty(this, Symbol.metadata, { enumerable: true, configurable: true, writable: true, value: _metadata });
         }
@@ -482,7 +542,10 @@ let Scraper = (() => {
          * - `type: "error"` with `error` details when scraping fails
          *
          * Sub-resources are collected via the `resourceResponse` event and
-         * included in the returned `ScrapeResult.resources`.
+         * included in the returned `ScrapeResult.resources`. Console messages and
+         * uncaught page errors (internal pages only) are collected via
+         * Puppeteer's `console`/`pageerror` page events and included in
+         * `ScrapeResult.consoleLogs`.
          * @param page - The Puppeteer page instance to use for navigation and DOM evaluation.
          * @param url - The extended URL to scrape.
          * @param options - Optional scraper configuration overriding defaults.
@@ -497,6 +560,7 @@ let Scraper = (() => {
             const metadataOnly = options?.metadataOnly ?? false;
             const imageLoadTimeout = options?.imageLoadTimeout ?? 5000;
             const resources = [];
+            const consoleLogs = [];
             const failedRequests = [];
             void this.emit('changePhase', {
                 pid: process.pid,
@@ -517,6 +581,7 @@ let Scraper = (() => {
                 return {
                     type: 'skipped',
                     resources,
+                    consoleLogs,
                     ignored: {
                         url,
                         matchedText: url.pathname || '',
@@ -540,6 +605,8 @@ let Scraper = (() => {
                     imageList: [],
                     anchorList: [],
                     html: '',
+                    mainContents: null,
+                    scrollHeight: null,
                     isSkipped: false,
                 };
                 void this.emit('changePhase', {
@@ -549,7 +616,7 @@ let Scraper = (() => {
                     isExternal,
                     message: '',
                 });
-                return { type: 'success', pageData: result, resources };
+                return { type: 'success', pageData: result, resources, consoleLogs };
             }
             let headResult = options?.headCheckResult ?? null;
             if (headResult && metadataOnly) {
@@ -565,12 +632,15 @@ let Scraper = (() => {
                     pageData: {
                         ...headResult,
                         isTarget: false,
+                        mainContents: headResult.mainContents ?? null,
+                        scrollHeight: headResult.scrollHeight ?? null,
                     },
                     resources,
+                    consoleLogs,
                 };
             }
-            if (headResult === null || headResult.contentType === 'text/html') {
-                const fetchResult = await this.#fetchData(page, url, isExternal, captureImages, imageLoadTimeout, resources, failedRequests, options).catch((error) => {
+            if (headResult === null || isHtmlContentType(headResult.contentType)) {
+                const fetchResult = await this.#fetchData(page, url, isExternal, captureImages, imageLoadTimeout, resources, consoleLogs, failedRequests, options).catch((error) => {
                     if (error instanceof Error) {
                         return error;
                     }
@@ -582,6 +652,7 @@ let Scraper = (() => {
                     return {
                         type: 'error',
                         resources,
+                        consoleLogs,
                         failedRequests: failedRequests.length > 0 ? failedRequests : undefined,
                         error: {
                             name: fetchResult.name,
@@ -612,6 +683,7 @@ let Scraper = (() => {
                         return {
                             type: 'skipped',
                             resources,
+                            consoleLogs,
                             ignored: {
                                 url,
                                 matchedText: url.pathname || '',
@@ -629,6 +701,7 @@ let Scraper = (() => {
                     return {
                         type: 'skipped',
                         resources,
+                        consoleLogs,
                         ignored: {
                             url,
                             matchedText: headResult.matched.text,
@@ -648,6 +721,7 @@ let Scraper = (() => {
                 type: 'success',
                 pageData: headResult,
                 resources,
+                consoleLogs,
                 failedRequests: failedRequests.length > 0 ? failedRequests : undefined,
             };
         }
@@ -703,7 +777,8 @@ let Scraper = (() => {
          * Frame" or "Session closed".
          *
          * Flow:
-         * 1. Register request/response/requestfailed listeners to capture sub-resources (internal pages only)
+         * 1. Register request/response/requestfailed/console/pageerror listeners to
+         *    capture sub-resources and console output (internal pages only)
          * 2. Navigate to URL via `page.goto()` and track redirect chain
          * 3. Wait for DOM content and network idle
          * 4. Check for network disconnection errors and throw to trigger retry
@@ -715,6 +790,7 @@ let Scraper = (() => {
          * @param captureImages - Whether to run the image extraction pipeline
          * @param imageLoadTimeout - Timeout (ms) for waiting lazy-loaded images to complete
          * @param resources - Mutable array to collect captured sub-resources into
+         * @param consoleLogs - Mutable array to collect captured console messages / page errors into
          * @param failedRequests - Mutable array to collect failed sub-resource requests into
          * @param options - Additional scraper options (e.g. `disableQueries`, `navigationTimeout`)
          * @returns Full page data or skipped page data if an exclusion rule matched
@@ -732,12 +808,12 @@ let Scraper = (() => {
          * changes and triggers a reload. Isolating each device preset allows partial
          * results — if one viewport fails, the other can still succeed.
          *
-         * WHY retryable with 20-min timeout and `fallback: []`: Image extraction is
-         * best-effort. If all retries fail, an empty array is returned rather than
-         * failing the entire page scrape. The 20-min wall clock accommodates pages
-         * whose mobile-small `scrollHeight` reaches ~300k px (observed on
-         * responsive data tables, which take ~5 min to scroll). A shorter timeout
-         * causes a second retry to start while the previous attempt's
+         * WHY retryable with 20-min timeout and empty fallback: Image extraction is
+         * best-effort. If all retries fail, empty images and null scroll heights are
+         * returned rather than failing the entire page scrape. The 20-min wall clock
+         * accommodates pages whose mobile-small `scrollHeight` reaches ~300k px
+         * (observed on responsive data tables, which take ~5 min to scroll). A shorter
+         * timeout causes a second retry to start while the previous attempt's
          * `scrollAllOver` is still running its `page.evaluate` calls in the
          * background — `Promise.race` in `retry.ts` does not cancel `fn()`. The
          * collision then surfaces as "Attempted to use detached Frame" or
@@ -749,12 +825,15 @@ let Scraper = (() => {
          * preset entirely keeps the timeout-vs-background-evaluate collision from
          * ever being triggered, at the cost of losing that viewport's image data
          * for those pages. See {@link MAX_SCROLL_HEIGHT} for the chosen threshold.
+         *
+         * Scroll heights are still recorded when scrolling is skipped for the
+         * height limit — the measurement from `beforePageScan` is kept.
          * @param page - Puppeteer page instance
          * @param url - The page URL string (without hash and auth)
          * @param isExternal - Whether the page is external
          * @param imageLoadTimeout - Timeout (ms) for waiting images to complete loading
          * @param domEvaluationTimeout - Timeout (ms) for the in-page image extraction `page.evaluate`
-         * @returns Array of image elements from all device presets (may be partial if some viewports failed)
+         * @returns Image elements plus desktop/mobile scroll heights from the scan path
          */
         get #fetchImages() { return _private_fetchImages_descriptor.value; }
     };

package/dist/to-console-log-entry.d.ts

@@ -0,0 +1,14 @@
+import type { ConsoleLogEntry } from './types.js';
+import type { ConsoleMessage } from 'puppeteer';
+/**
+ * Converts a Puppeteer `ConsoleMessage` into a `ConsoleLogEntry`.
+ *
+ * WHY per-argument try/catch: `JSHandle.jsonValue()` can reject when the
+ * page's execution context has been destroyed (e.g. mid-navigation) or when
+ * the value cannot be serialized (e.g. a circular reference). A single
+ * failing argument must not drop the rest of the message's arguments.
+ * @param msg - The console message captured via `page.on('console')`
+ * @param pageUrl - The URL (without hash) of the page that produced the message
+ * @returns The resolved console log entry
+ */
+export declare function toConsoleLogEntry(msg: ConsoleMessage, pageUrl: string): Promise<ConsoleLogEntry>;

package/dist/to-console-log-entry.js

@@ -0,0 +1,33 @@
+/**
+ * Converts a Puppeteer `ConsoleMessage` into a `ConsoleLogEntry`.
+ *
+ * WHY per-argument try/catch: `JSHandle.jsonValue()` can reject when the
+ * page's execution context has been destroyed (e.g. mid-navigation) or when
+ * the value cannot be serialized (e.g. a circular reference). A single
+ * failing argument must not drop the rest of the message's arguments.
+ * @param msg - The console message captured via `page.on('console')`
+ * @param pageUrl - The URL (without hash) of the page that produced the message
+ * @returns The resolved console log entry
+ */
+export async function toConsoleLogEntry(msg, pageUrl) {
+    const args = await Promise.all(msg.args().map(async (arg) => {
+        try {
+            return await arg.jsonValue();
+        }
+        catch {
+            return;
+        }
+        finally {
+            await arg.dispose().catch(() => { });
+        }
+    }));
+    const loc = msg.location();
+    return {
+        pageUrl,
+        type: msg.type(),
+        text: msg.text(),
+        args,
+        location: loc.url === undefined ? undefined : loc,
+        ts: Date.now(),
+    };
+}