@d-zero/beholder 3.1.5 → 4.0.0

package/dist/scraper.js

@@ -43,8 +43,11 @@ import { retry as retryable } from '@d-zero/shared/retry';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
 import { resourceLog, scraperLog } from './debug.js';
 import { DEFAULT_DOM_EVALUATION_TIMEOUT, getAnchorList, getImageList, getMeta, } from './dom-evaluation.js';
+import { getMainContents } from './get-main-contents.js';
 import { isError } from './is-error.js';
+import { isHtmlContentType } from './is-html-content-type.js';
 import { keywordCheck } from './keyword-check.js';
+import { measureScrollHeight } from './measure-scroll-height.js';
 import { emptyMeta } from './meta/classify.js';
 import { findDisconnectionFailures } from './network-disconnection.js';
 import { parseUrl } from './parse-url.js';
@@ -95,7 +98,10 @@ let Scraper = (() => {
                 })];
             _private_fetchImages_decorators = [retryable({
                     timeout: 20 * 60 * 1000,
-                    fallback: [],
+                    fallback: {
+                        imageList: [],
+                        scrollHeight: { desktop: null, mobile: null },
+                    },
                     onWait(determinedInterval, retryCount, methodName, error) {
                         void this.emit('changePhase', {
                             pid: process.pid,
@@ -267,7 +273,7 @@ let Scraper = (() => {
                     const contentType = responseHeaders['content-type']?.split(';')[0] || null;
                     const _contentLength = Number.parseInt(responseHeaders['content-length'] ?? '');
                     const contentLength = Number.isFinite(_contentLength) ? _contentLength : null;
-                    if (contentType !== 'text/html') {
+                    if (!isHtmlContentType(contentType)) {
                         return {
                             url,
                             isTarget: false,
@@ -282,6 +288,8 @@ let Scraper = (() => {
                             imageList: [],
                             anchorList: [],
                             html: '',
+                            mainContents: null,
+                            scrollHeight: null,
                             isSkipped: false,
                         };
                     }
@@ -326,6 +334,8 @@ let Scraper = (() => {
                             imageList: [],
                             anchorList: [],
                             html,
+                            mainContents: null,
+                            scrollHeight: null,
                             isSkipped: false,
                         };
                     }
@@ -347,6 +357,9 @@ let Scraper = (() => {
                             .join(', ');
                         throw new Error(`Network disconnection detected during page load: ${errorSummary}`);
                     }
+                    const mainContents = await getMainContents(page, {
+                        mainContentSelector: options?.mainContentSelector,
+                    });
                     void this.emit('changePhase', {
                         pid: process.pid,
                         name: 'getAnchors',
@@ -368,18 +381,23 @@ let Scraper = (() => {
                         statusCode: status,
                         headers: responseHeaders ?? undefined,
                     }, domEvaluationTimeout);
-                    const imageList = captureImages
-                        ? await (async () => {
-                            void this.emit('changePhase', {
-                                pid: process.pid,
-                                name: 'extractImages',
-                                url,
-                                isExternal,
-                                message: `%countdown(${domEvaluationTimeout},extractImages_${url.withoutHash},s)%s`,
-                            });
-                            return this.#fetchImages(page, url.withoutHashAndAuth, isExternal, imageLoadTimeout, domEvaluationTimeout);
-                        })()
-                        : [];
+                    let imageList = [];
+                    let scrollHeight = null;
+                    if (captureImages) {
+                        void this.emit('changePhase', {
+                            pid: process.pid,
+                            name: 'extractImages',
+                            url,
+                            isExternal,
+                            message: `%countdown(${domEvaluationTimeout},extractImages_${url.withoutHash},s)%s`,
+                        });
+                        const fetched = await this.#fetchImages(page, url.withoutHashAndAuth, isExternal, imageLoadTimeout, domEvaluationTimeout);
+                        imageList = fetched.imageList;
+                        scrollHeight = fetched.scrollHeight;
+                    }
+                    else {
+                        scrollHeight = await measureScrollHeight(page);
+                    }
                     return {
                         url,
                         isTarget: true,
@@ -394,6 +412,8 @@ let Scraper = (() => {
                         anchorList,
                         imageList,
                         html,
+                        mainContents,
+                        scrollHeight,
                         isSkipped: false,
                     };
                 }, "#fetchData") }, _private_fetchData_decorators, { kind: "method", name: "#fetchData", static: false, private: true, access: { has: obj => #fetchData in obj, get: obj => obj.#fetchData }, metadata: _metadata }, null, _instanceExtraInitializers);
@@ -404,7 +424,9 @@ let Scraper = (() => {
                         { key: 'mobile-small', preset: devicePresets['mobile-small'] },
                     ];
                     const imageList = [];
+                    const scrollHeight = { desktop: null, mobile: null };
                     for (const { key, preset } of devices) {
+                        const scrollKey = key === 'desktop-compact' ? 'desktop' : 'mobile';
                         try {
                             void this.emit('changePhase', {
                                 pid: process.pid,
@@ -421,6 +443,7 @@ let Scraper = (() => {
                                 timeout: 5000,
                                 maxScrollHeight: MAX_SCROLL_HEIGHT,
                             });
+                            scrollHeight[scrollKey] = scanResult.scrollHeight;
                             if (!scanResult.scrolled) {
                                 void this.emit('changePhase', {
                                     pid: process.pid,
@@ -465,7 +488,7 @@ let Scraper = (() => {
                             });
                         }
                     }
-                    return imageList;
+                    return { imageList, scrollHeight };
                 }, "#fetchImages") }, _private_fetchImages_decorators, { kind: "method", name: "#fetchImages", static: false, private: true, access: { has: obj => #fetchImages in obj, get: obj => obj.#fetchImages }, metadata: _metadata }, null, _instanceExtraInitializers);
             if (_metadata) Object.defineProperty(this, Symbol.metadata, { enumerable: true, configurable: true, writable: true, value: _metadata });
         }
@@ -540,6 +563,8 @@ let Scraper = (() => {
                     imageList: [],
                     anchorList: [],
                     html: '',
+                    mainContents: null,
+                    scrollHeight: null,
                     isSkipped: false,
                 };
                 void this.emit('changePhase', {
@@ -565,11 +590,13 @@ let Scraper = (() => {
                     pageData: {
                         ...headResult,
                         isTarget: false,
+                        mainContents: headResult.mainContents ?? null,
+                        scrollHeight: headResult.scrollHeight ?? null,
                     },
                     resources,
                 };
             }
-            if (headResult === null || headResult.contentType === 'text/html') {
+            if (headResult === null || isHtmlContentType(headResult.contentType)) {
                 const fetchResult = await this.#fetchData(page, url, isExternal, captureImages, imageLoadTimeout, resources, failedRequests, options).catch((error) => {
                     if (error instanceof Error) {
                         return error;
@@ -732,12 +759,12 @@ let Scraper = (() => {
          * changes and triggers a reload. Isolating each device preset allows partial
          * results — if one viewport fails, the other can still succeed.
          *
-         * WHY retryable with 20-min timeout and `fallback: []`: Image extraction is
-         * best-effort. If all retries fail, an empty array is returned rather than
-         * failing the entire page scrape. The 20-min wall clock accommodates pages
-         * whose mobile-small `scrollHeight` reaches ~300k px (observed on
-         * responsive data tables, which take ~5 min to scroll). A shorter timeout
-         * causes a second retry to start while the previous attempt's
+         * WHY retryable with 20-min timeout and empty fallback: Image extraction is
+         * best-effort. If all retries fail, empty images and null scroll heights are
+         * returned rather than failing the entire page scrape. The 20-min wall clock
+         * accommodates pages whose mobile-small `scrollHeight` reaches ~300k px
+         * (observed on responsive data tables, which take ~5 min to scroll). A shorter
+         * timeout causes a second retry to start while the previous attempt's
          * `scrollAllOver` is still running its `page.evaluate` calls in the
          * background — `Promise.race` in `retry.ts` does not cancel `fn()`. The
          * collision then surfaces as "Attempted to use detached Frame" or
@@ -749,12 +776,15 @@ let Scraper = (() => {
          * preset entirely keeps the timeout-vs-background-evaluate collision from
          * ever being triggered, at the cost of losing that viewport's image data
          * for those pages. See {@link MAX_SCROLL_HEIGHT} for the chosen threshold.
+         *
+         * Scroll heights are still recorded when scrolling is skipped for the
+         * height limit — the measurement from `beforePageScan` is kept.
          * @param page - Puppeteer page instance
          * @param url - The page URL string (without hash and auth)
          * @param isExternal - Whether the page is external
          * @param imageLoadTimeout - Timeout (ms) for waiting images to complete loading
          * @param domEvaluationTimeout - Timeout (ms) for the in-page image extraction `page.evaluate`
-         * @returns Array of image elements from all device presets (may be partial if some viewports failed)
+         * @returns Image elements plus desktop/mobile scroll heights from the scan path
          */
         get #fetchImages() { return _private_fetchImages_descriptor.value; }
     };

package/dist/types.d.ts

@@ -42,9 +42,187 @@ export type PageData = {
     imageList: ImageElement[];
     /** HTML snapshot of the rendered DOM. */
     html: string;
+    /**
+     * Quantitative main-content metrics extracted in the browser after load.
+     * `null` when the page is non-HTML, external, non-HTTP, or otherwise not fully scraped.
+     * When HTML is scraped but no main region is found, this is still an object with empty arrays and `wordCount: 0`.
+     */
+    mainContents: MainContentsData | null;
+    /**
+     * `document.body.scrollHeight` at desktop-compact and mobile-small viewports.
+     * `null` when not measured (non-HTML, external, non-HTTP, or skipped).
+     */
+    scrollHeight: ScrollHeightData | null;
     /** Always `false` for successfully scraped pages. See {@link SkippedPageData} for skipped pages. */
     isSkipped: false;
 };
+/**
+ * `document.body.scrollHeight` measured at the scraper's desktop and mobile device presets.
+ */
+export type ScrollHeightData = {
+    /** Height at `desktop-compact` (width 1280), or `null` if that preset failed. */
+    desktop: number | null;
+    /** Height at `mobile-small` (width 320 @ 2x), or `null` if that preset failed. */
+    mobile: number | null;
+};
+/**
+ * Quantitative metrics for the detected main content region of a page.
+ * @example
+ * ```ts
+ * const { mainContents } = pageData;
+ * if (mainContents) {
+ *   console.log(mainContents.wordCount, mainContents.headings.length);
+ * }
+ * ```
+ */
+export type MainContentsData = {
+    /** `document.title` trimmed. Empty string when the title is empty. */
+    title: string;
+    /**
+     * Identifying info for the detected main region element.
+     * `null` when no selector matched (arrays are empty and `wordCount` is `0`).
+     */
+    main: MainContentsMainTag | null;
+    /**
+     * Character count of the main region's `textContent` after whitespace removal.
+     * `0` when no main region was found or the text is whitespace-only.
+     */
+    wordCount: number;
+    /**
+     * Character count of `document.body` `textContent` after whitespace removal.
+     * Measured even when no main region is found (`0` if `body` is missing).
+     */
+    bodyWordCount: number;
+    /** Headings (`h1`–`h6`) inside the main region, in DOM order. */
+    headings: MainContentsHeading[];
+    /** Images (`img`, `input[type=image]`) inside the main region, in DOM order. */
+    images: MainContentsImage[];
+    /** Tables inside the main region, in DOM order. */
+    tables: MainContentsTable[];
+    /** Button-like elements inside the main region, in DOM order. */
+    buttons: MainContentsButton[];
+    /** Iframes inside the main region, in DOM order. */
+    iframes: MainContentsIframe[];
+    /** Videos inside the main region, in DOM order. */
+    videos: MainContentsVideo[];
+    /** Audios inside the main region, in DOM order. */
+    audios: MainContentsAudio[];
+    /** Canvases inside the main region, in DOM order. */
+    canvases: MainContentsCanvas[];
+};
+/**
+ * Identifying information about the DOM element used as the main content area.
+ */
+export type MainContentsMainTag = {
+    /** Element tag name (e.g. `"MAIN"`, `"DIV"`), as reported by `nodeName`. */
+    nodeName: string;
+    /** The element's `id`, or `null` when empty/absent. */
+    id: string | null;
+    /** CSS class names on the element. */
+    classList: string[];
+    /** The WAI-ARIA `role` attribute value, or `null` when absent. */
+    role: string | null;
+    /**
+     * A simple diagnostic CSS selector built from tag + id + classes
+     * (not `@medv/finder`). // cspell:disable-line
+     */
+    selector: string;
+};
+/**
+ * A heading element (`h1`–`h6`) found within the main content area.
+ */
+export type MainContentsHeading = {
+    /** Heading text after whitespace removal, or `null` when empty. */
+    text: string | null;
+    /** Heading level from the tag name. */
+    level: 1 | 2 | 3 | 4 | 5 | 6;
+};
+/**
+ * An `<img>` or `<input type="image">` element found in the main content.
+ */
+export type MainContentsImage = {
+    /** Resolved absolute `src` URL. */
+    src: string;
+    /** `alt` attribute value (may be an empty string). */
+    alt: string;
+};
+/**
+ * Structural summary of a `<table>` found within the main content.
+ */
+export type MainContentsTable = {
+    /** Number of `<tr>` elements. */
+    rows: number;
+    /** Number of `th`/`td` cells in the first row, or `0` when there is no row. */
+    cols: number;
+    /** Whether the table contains a `<thead>`. */
+    hasHeader: boolean;
+    /** Whether the table contains a `<tfoot>`. */
+    hasFooter: boolean;
+    /** Whether any cell uses `colspan` or `rowspan`. */
+    hasMergedCell: boolean;
+};
+/**
+ * A button-like element found in the main content
+ * (`button`, `[role=button]`, `[class*=button]`, `[class*=btn]`).
+ */
+export type MainContentsButton = {
+    /** Element tag name (e.g. `"BUTTON"`, `"A"`, `"DIV"`). */
+    nodeName: string;
+    /** `role` attribute, or `null` when absent. */
+    role: string | null;
+    /** `type` for `<button>` / `<input>`, otherwise `null`. */
+    type: string | null;
+    /** Label text after whitespace removal, or `null` when empty. */
+    text: string | null;
+    /** `true` when `disabled` or `aria-disabled="true"`. */
+    disabled: boolean;
+};
+/**
+ * An `<iframe>` found in the main content.
+ */
+export type MainContentsIframe = {
+    /** Resolved absolute `src` URL (empty string when unset). */
+    src: string;
+    /** `title` attribute, or `null` when the attribute is absent. */
+    title: string | null;
+    /** Raw `width` attribute string, or `null` when absent. */
+    width: string | null;
+    /** Raw `height` attribute string, or `null` when absent. */
+    height: string | null;
+};
+/**
+ * A `<video>` found in the main content.
+ */
+export type MainContentsVideo = {
+    /**
+     * Resolved media URL: `currentSrc`, else `src` attribute, else first `source[src]`, else `""`.
+     */
+    src: string;
+    /** Resolved `poster` URL, or `null` when unset. */
+    poster: string | null;
+    /** IDL `width` in pixels. */
+    width: number;
+    /** IDL `height` in pixels. */
+    height: number;
+};
+/**
+ * An `<audio>` found in the main content.
+ */
+export type MainContentsAudio = {
+    /**
+     * Resolved media URL: `currentSrc`, else `src` attribute, else first `source[src]`, else `""`.
+     */
+    src: string;
+};
+/**
+ * A `<canvas>` found in the main content.
+ */
+export type MainContentsCanvas = {
+    /** IDL bitmap width. */
+    width: number;
+    /** IDL bitmap height. */
+    height: number;
+};
 /**
  * Information about an image element found on a page.
  */
@@ -316,4 +494,9 @@ export type ScraperOptions = {
      * Default: 180_000 (180s, aligned with the upstream retryable timeout).
      */
     domEvaluationTimeout?: number;
+    /**
+     * CSS selector prepended to the default main-content detection list
+     * (highest priority). When omitted or `null`, only the built-in selectors are used.
+     */
+    mainContentSelector?: string | null;
 };

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@d-zero/beholder",
-	"version": "3.1.5",
+	"version": "4.0.0",
 	"description": "Page-level scraper for web crawling and auditing",
 	"author": "D-ZERO",
 	"license": "MIT",
@@ -20,7 +20,7 @@
 		"clean": "tsc --build --clean"
 	},
 	"dependencies": {
-		"@d-zero/puppeteer-page-scan": "4.6.4",
+		"@d-zero/puppeteer-page-scan": "4.6.5",
 		"@d-zero/shared": "0.22.2",
 		"debug": "4.4.3",
 		"puppeteer": "25.3.0",
@@ -36,5 +36,5 @@
 		"url": "https://github.com/d-zero-dev/tools.git",
 		"directory": "packages/@d-zero/beholder"
 	},
-	"gitHead": "d5bd3c24d9ad6e422d92385645c6f0b3fd3a5ee8"
+	"gitHead": "581df4a1796bd864f30980c759e6ada99914ffb7"
 }