@d-zero/beholder 3.1.6 → 4.1.0

package/dist/to-page-error-entry.d.ts

@@ -0,0 +1,10 @@
+import type { ConsoleLogEntry } from './types.js';
+/**
+ * Converts an uncaught exception / unhandled Promise rejection (captured via
+ * `page.on('pageerror')`) into a `ConsoleLogEntry`.
+ * @param error - The value emitted by `page.on('pageerror')`; typed as `unknown`
+ * because Puppeteer allows non-`Error` throw values (e.g. `throw 'oops'`)
+ * @param pageUrl - The URL (without hash) of the page that produced the error
+ * @returns The console log entry, with `type: 'pageerror'`
+ */
+export declare function toPageErrorEntry(error: unknown, pageUrl: string): ConsoleLogEntry;

package/dist/to-page-error-entry.js

@@ -0,0 +1,19 @@
+/**
+ * Converts an uncaught exception / unhandled Promise rejection (captured via
+ * `page.on('pageerror')`) into a `ConsoleLogEntry`.
+ * @param error - The value emitted by `page.on('pageerror')`; typed as `unknown`
+ * because Puppeteer allows non-`Error` throw values (e.g. `throw 'oops'`)
+ * @param pageUrl - The URL (without hash) of the page that produced the error
+ * @returns The console log entry, with `type: 'pageerror'`
+ */
+export function toPageErrorEntry(error, pageUrl) {
+    const isErr = error instanceof Error;
+    return {
+        pageUrl,
+        type: 'pageerror',
+        text: isErr ? error.message : String(error),
+        args: [],
+        stack: isErr ? error.stack : undefined,
+        ts: Date.now(),
+    };
+}

package/dist/types.d.ts

@@ -12,6 +12,7 @@ import type { Meta } from './meta/types.js';
 import type { CDNType } from '@d-zero/shared/detect-cdn';
 import type { CompressType } from '@d-zero/shared/detect-compress';
 import type { ExURL } from '@d-zero/shared/parse-url';
+import type { ConsoleMessageType } from 'puppeteer';
 /**
  * Scraped page data returned by the scraper after successfully processing a page.
  */
@@ -42,9 +43,187 @@ export type PageData = {
     imageList: ImageElement[];
     /** HTML snapshot of the rendered DOM. */
     html: string;
+    /**
+     * Quantitative main-content metrics extracted in the browser after load.
+     * `null` when the page is non-HTML, external, non-HTTP, or otherwise not fully scraped.
+     * When HTML is scraped but no main region is found, this is still an object with empty arrays and `wordCount: 0`.
+     */
+    mainContents: MainContentsData | null;
+    /**
+     * `document.body.scrollHeight` at desktop-compact and mobile-small viewports.
+     * `null` when not measured (non-HTML, external, non-HTTP, or skipped).
+     */
+    scrollHeight: ScrollHeightData | null;
     /** Always `false` for successfully scraped pages. See {@link SkippedPageData} for skipped pages. */
     isSkipped: false;
 };
+/**
+ * `document.body.scrollHeight` measured at the scraper's desktop and mobile device presets.
+ */
+export type ScrollHeightData = {
+    /** Height at `desktop-compact` (width 1280), or `null` if that preset failed. */
+    desktop: number | null;
+    /** Height at `mobile-small` (width 320 @ 2x), or `null` if that preset failed. */
+    mobile: number | null;
+};
+/**
+ * Quantitative metrics for the detected main content region of a page.
+ * @example
+ * ```ts
+ * const { mainContents } = pageData;
+ * if (mainContents) {
+ *   console.log(mainContents.wordCount, mainContents.headings.length);
+ * }
+ * ```
+ */
+export type MainContentsData = {
+    /** `document.title` trimmed. Empty string when the title is empty. */
+    title: string;
+    /**
+     * Identifying info for the detected main region element.
+     * `null` when no selector matched (arrays are empty and `wordCount` is `0`).
+     */
+    main: MainContentsMainTag | null;
+    /**
+     * Character count of the main region's `textContent` after whitespace removal.
+     * `0` when no main region was found or the text is whitespace-only.
+     */
+    wordCount: number;
+    /**
+     * Character count of `document.body` `textContent` after whitespace removal.
+     * Measured even when no main region is found (`0` if `body` is missing).
+     */
+    bodyWordCount: number;
+    /** Headings (`h1`–`h6`) inside the main region, in DOM order. */
+    headings: MainContentsHeading[];
+    /** Images (`img`, `input[type=image]`) inside the main region, in DOM order. */
+    images: MainContentsImage[];
+    /** Tables inside the main region, in DOM order. */
+    tables: MainContentsTable[];
+    /** Button-like elements inside the main region, in DOM order. */
+    buttons: MainContentsButton[];
+    /** Iframes inside the main region, in DOM order. */
+    iframes: MainContentsIframe[];
+    /** Videos inside the main region, in DOM order. */
+    videos: MainContentsVideo[];
+    /** Audios inside the main region, in DOM order. */
+    audios: MainContentsAudio[];
+    /** Canvases inside the main region, in DOM order. */
+    canvases: MainContentsCanvas[];
+};
+/**
+ * Identifying information about the DOM element used as the main content area.
+ */
+export type MainContentsMainTag = {
+    /** Element tag name (e.g. `"MAIN"`, `"DIV"`), as reported by `nodeName`. */
+    nodeName: string;
+    /** The element's `id`, or `null` when empty/absent. */
+    id: string | null;
+    /** CSS class names on the element. */
+    classList: string[];
+    /** The WAI-ARIA `role` attribute value, or `null` when absent. */
+    role: string | null;
+    /**
+     * A simple diagnostic CSS selector built from tag + id + classes
+     * (not `@medv/finder`). // cspell:disable-line
+     */
+    selector: string;
+};
+/**
+ * A heading element (`h1`–`h6`) found within the main content area.
+ */
+export type MainContentsHeading = {
+    /** Heading text after whitespace removal, or `null` when empty. */
+    text: string | null;
+    /** Heading level from the tag name. */
+    level: 1 | 2 | 3 | 4 | 5 | 6;
+};
+/**
+ * An `<img>` or `<input type="image">` element found in the main content.
+ */
+export type MainContentsImage = {
+    /** Resolved absolute `src` URL. */
+    src: string;
+    /** `alt` attribute value (may be an empty string). */
+    alt: string;
+};
+/**
+ * Structural summary of a `<table>` found within the main content.
+ */
+export type MainContentsTable = {
+    /** Number of `<tr>` elements. */
+    rows: number;
+    /** Number of `th`/`td` cells in the first row, or `0` when there is no row. */
+    cols: number;
+    /** Whether the table contains a `<thead>`. */
+    hasHeader: boolean;
+    /** Whether the table contains a `<tfoot>`. */
+    hasFooter: boolean;
+    /** Whether any cell uses `colspan` or `rowspan`. */
+    hasMergedCell: boolean;
+};
+/**
+ * A button-like element found in the main content
+ * (`button`, `[role=button]`, `[class*=button]`, `[class*=btn]`).
+ */
+export type MainContentsButton = {
+    /** Element tag name (e.g. `"BUTTON"`, `"A"`, `"DIV"`). */
+    nodeName: string;
+    /** `role` attribute, or `null` when absent. */
+    role: string | null;
+    /** `type` for `<button>` / `<input>`, otherwise `null`. */
+    type: string | null;
+    /** Label text after whitespace removal, or `null` when empty. */
+    text: string | null;
+    /** `true` when `disabled` or `aria-disabled="true"`. */
+    disabled: boolean;
+};
+/**
+ * An `<iframe>` found in the main content.
+ */
+export type MainContentsIframe = {
+    /** Resolved absolute `src` URL (empty string when unset). */
+    src: string;
+    /** `title` attribute, or `null` when the attribute is absent. */
+    title: string | null;
+    /** Raw `width` attribute string, or `null` when absent. */
+    width: string | null;
+    /** Raw `height` attribute string, or `null` when absent. */
+    height: string | null;
+};
+/**
+ * A `<video>` found in the main content.
+ */
+export type MainContentsVideo = {
+    /**
+     * Resolved media URL: `currentSrc`, else `src` attribute, else first `source[src]`, else `""`.
+     */
+    src: string;
+    /** Resolved `poster` URL, or `null` when unset. */
+    poster: string | null;
+    /** IDL `width` in pixels. */
+    width: number;
+    /** IDL `height` in pixels. */
+    height: number;
+};
+/**
+ * An `<audio>` found in the main content.
+ */
+export type MainContentsAudio = {
+    /**
+     * Resolved media URL: `currentSrc`, else `src` attribute, else first `source[src]`, else `""`.
+     */
+    src: string;
+};
+/**
+ * A `<canvas>` found in the main content.
+ */
+export type MainContentsCanvas = {
+    /** IDL bitmap width. */
+    width: number;
+    /** IDL bitmap height. */
+    height: number;
+};
 /**
  * Information about an image element found on a page.
  */
@@ -189,6 +368,8 @@ export type ScrapeResult = {
     pageData?: PageData;
     /** All sub-resources captured during the page load. */
     resources: ResourceEntry[];
+    /** All console messages and uncaught page errors captured during the page load. */
+    consoleLogs: ConsoleLogEntry[];
     /** Details about why the page was ignored, present when `type` is `"skipped"`. */
     ignored?: {
         url: ExURL;
@@ -220,6 +401,38 @@ export type ResourceEntry = {
     /** The URL (without hash) of the page that triggered this resource load. */
     pageUrl: string;
 };
+/**
+ * A single console message or uncaught page error captured during page scraping.
+ * Captured via Puppeteer's `console` and `pageerror` page events; internal pages only.
+ */
+export type ConsoleLogEntry = {
+    /** The URL (without hash) of the page that produced this message. */
+    pageUrl: string;
+    /**
+     * The console message type (Puppeteer's `ConsoleMessageType`, e.g. `"log"`, `"warn"`, `"error"`),
+     * or `"pageerror"` for an uncaught exception / unhandled Promise rejection.
+     */
+    type: ConsoleMessageType | 'pageerror';
+    /** The message text, or the error's `message` for `"pageerror"` entries. */
+    text: string;
+    /**
+     * Arguments passed to the console call, resolved via `JSHandle.jsonValue()`.
+     * An individual argument is `undefined` when it could not be resolved (e.g. a
+     * destroyed execution context or a value `jsonValue()` cannot serialize).
+     * Always empty for `"pageerror"` entries.
+     */
+    args: unknown[];
+    /** Source location of the message, or `undefined` when unavailable (always `undefined` for `"pageerror"`). */
+    location?: {
+        url?: string;
+        lineNumber?: number;
+        columnNumber?: number;
+    };
+    /** Stack trace text, present only for `"pageerror"` entries. */
+    stack?: string;
+    /** Timestamp (ms since epoch) when the message was captured. */
+    ts: number;
+};
 /**
  * Event payload describing a phase transition in the scraping lifecycle.
  * Phases proceed roughly in order: scrapeStart -> openPage ->
@@ -316,4 +529,9 @@ export type ScraperOptions = {
      * Default: 180_000 (180s, aligned with the upstream retryable timeout).
      */
     domEvaluationTimeout?: number;
+    /**
+     * CSS selector prepended to the default main-content detection list
+     * (highest priority). When omitted or `null`, only the built-in selectors are used.
+     */
+    mainContentSelector?: string | null;
 };

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@d-zero/beholder",
-	"version": "3.1.6",
+	"version": "4.1.0",
 	"description": "Page-level scraper for web crawling and auditing",
 	"author": "D-ZERO",
 	"license": "MIT",
@@ -36,5 +36,5 @@
 		"url": "https://github.com/d-zero-dev/tools.git",
 		"directory": "packages/@d-zero/beholder"
 	},
-	"gitHead": "9c1e3da176d39972eae9903a979e942a59515229"
+	"gitHead": "d8d1cf278af98e1779e270ddd08da9496eae71b1"
 }

package/src/get-main-contents.spec.ts

@@ -0,0 +1,282 @@
+import type { Page } from 'puppeteer';
+
+import { JSDOM } from 'jsdom';
+import { describe, expect, it, vi } from 'vitest';
+
+import { extractMainContentsFromDocument, getMainContents } from './get-main-contents.js';
+
+/**
+ * @param html
+ */
+function createDocument(html: string): Document {
+	return new JSDOM(html, { url: 'https://example.com/' }).window.document;
+}
+
+/**
+ * @param html
+ * @param mainContentSelector
+ */
+function extract(html: string, mainContentSelector: string | null = null) {
+	return extractMainContentsFromDocument(mainContentSelector, createDocument(html));
+}
+
+describe('extractMainContentsFromDocument', () => {
+	it('returns empty main metrics when no main content element is found, but counts body text', () => {
+		const result = extract(
+			'<html><head><title>T</title></head><body><header>ABC</header><div>No main</div></body></html>',
+		);
+
+		expect(result).toEqual({
+			title: 'T',
+			main: null,
+			wordCount: 0,
+			bodyWordCount: 9,
+			headings: [],
+			images: [],
+			tables: [],
+			buttons: [],
+			iframes: [],
+			videos: [],
+			audios: [],
+			canvases: [],
+		});
+	});
+
+	it('detects <main> and counts word length', () => {
+		const result = extract('<html><body><main>こんにちは世界</main></body></html>');
+
+		expect(result.wordCount).toBe(7);
+		expect(result.bodyWordCount).toBeGreaterThanOrEqual(7);
+		expect(result.main?.nodeName).toBe('MAIN');
+	});
+
+	it('detects [role="main"]', () => {
+		const result = extract('<html><body><div role="main">Content</div></body></html>');
+
+		expect(result.wordCount).toBe(7);
+		expect(result.main?.role).toBe('main');
+	});
+
+	it('uses custom mainContentSelector when provided', () => {
+		const result = extract(
+			'<html><body><section id="page-body">Custom main</section></body></html>',
+			'#page-body',
+		);
+
+		expect(result.wordCount).toBe(10);
+		expect(result.main?.id).toBe('page-body');
+	});
+
+	it('falls back to built-in selectors when custom selector is invalid', () => {
+		const result = extract(
+			'<html><body><main>Still works</main></body></html>',
+			'[[[invalid',
+		);
+
+		expect(result.main?.nodeName).toBe('MAIN');
+		expect(result.wordCount).toBe(10);
+	});
+
+	it('extracts headings from main content', () => {
+		const result = extract(`
+			<html><body><main>
+				<h1>Title</h1>
+				<h2>Subtitle</h2>
+				<h3>Section</h3>
+			</main></body></html>
+		`);
+
+		expect(result.headings).toEqual([
+			{ text: 'Title', level: 1 },
+			{ text: 'Subtitle', level: 2 },
+			{ text: 'Section', level: 3 },
+		]);
+	});
+
+	it('extracts images from main content', () => {
+		const result = extract(`
+			<html><body><main>
+				<img src="a.png" alt="Image A">
+				<img src="b.png" alt="Image B">
+				<input type="image" src="c.png" alt="Image C">
+			</main></body></html>
+		`);
+
+		expect(result.images).toHaveLength(3);
+		expect(result.images[0]?.alt).toBe('Image A');
+		expect(result.images[0]?.src).toContain('a.png');
+	});
+
+	it('extracts table metadata from main content', () => {
+		const result = extract(`
+			<html><body><main>
+				<table>
+					<thead><tr><th>A</th><th>B</th></tr></thead>
+					<tfoot><tr><td>Footer</td><td></td></tr></tfoot>
+					<tbody>
+						<tr><td colspan="2">Merged</td></tr>
+						<tr><td>1</td><td>2</td></tr>
+					</tbody>
+				</table>
+			</main></body></html>
+		`);
+
+		expect(result.tables).toEqual([
+			{
+				rows: 4,
+				cols: 2,
+				hasHeader: true,
+				hasFooter: true,
+				hasMergedCell: true,
+			},
+		]);
+	});
+
+	it('extracts buttons from main content', () => {
+		const result = extract(`
+			<html><body><main>
+				<button>Save</button>
+				<a role="button" href="#">Go</a>
+				<span class="btn">Click</span>
+				<div class="share-button">Share</div>
+			</main></body></html>
+		`);
+
+		expect(result.buttons).toHaveLength(4);
+		expect(result.buttons[0]).toMatchObject({
+			nodeName: 'BUTTON',
+			text: 'Save',
+			disabled: false,
+		});
+		expect(result.buttons[1]?.role).toBe('button');
+	});
+
+	it('extracts iframe, video, audio, and canvas from main content', () => {
+		const result = extract(`
+			<html><body><main>
+				<iframe src="/embed" title="Embed" width="640" height="360"></iframe>
+				<video src="/movie.mp4" poster="/poster.jpg" width="320" height="180"></video>
+				<audio src="/sound.mp3"></audio>
+				<canvas width="100" height="50"></canvas>
+			</main></body></html>
+		`);
+
+		expect(result.iframes).toHaveLength(1);
+		expect(result.iframes[0]).toMatchObject({
+			title: 'Embed',
+			width: '640',
+			height: '360',
+		});
+		expect(result.iframes[0]?.src).toContain('/embed');
+
+		expect(result.videos).toHaveLength(1);
+		expect(result.videos[0]?.src).toContain('/movie.mp4');
+		expect(result.videos[0]?.poster).toContain('/poster.jpg');
+		expect(result.videos[0]?.width).toBe(320);
+		expect(result.videos[0]?.height).toBe(180);
+
+		expect(result.audios).toHaveLength(1);
+		expect(result.audios[0]?.src).toContain('/sound.mp3');
+
+		expect(result.canvases).toEqual([{ width: 100, height: 50 }]);
+	});
+
+	it('treats whitespace-only main text as zero word count', () => {
+		const result = extract('<html><body><main>   \n\t  </main></body></html>');
+
+		expect(result.wordCount).toBe(0);
+	});
+
+	it('detects #content fallback selector', () => {
+		const result = extract(
+			'<html><body><div id="content">Fallback content</div></body></html>',
+		);
+
+		expect(result.wordCount).toBe(15);
+	});
+
+	it('detects .main fallback selector', () => {
+		const result = extract(
+			'<html><body><div class="main">Class-based main</div></body></html>',
+		);
+
+		expect(result.wordCount).toBe(15);
+	});
+
+	it('returns first matching element in DOM order when multiple Phase-1 selectors match', () => {
+		const result = extract(`
+			<html><body>
+				<main>Semantic main</main>
+				<div id="content">ID content</div>
+			</body></html>
+		`);
+
+		expect(result.main?.nodeName).toBe('MAIN');
+		expect(result.wordCount).toBe(12);
+	});
+
+	it('uses Phase-2 class*=main when Phase-1 finds nothing', () => {
+		const result = extract(
+			'<html><body><div class="page-main-area">Phase two</div></body></html>',
+		);
+
+		expect(result.main?.classList).toContain('page-main-area');
+		// "Phase two" → "Phasetwo" (8) after whitespace removal
+		expect(result.wordCount).toBe(8);
+	});
+
+	it('uses Phase-2 id*=content case-insensitively when Phase-1 finds nothing', () => {
+		const result = extract(
+			'<html><body><div id="PrimaryContent">Primary</div></body></html>',
+		);
+
+		expect(result.main?.id).toBe('PrimaryContent');
+		expect(result.wordCount).toBe(7);
+	});
+
+	it('prefers Phase-1 <main> over an earlier header-content that would match Phase-2', () => {
+		const result = extract(`
+			<html><body>
+				<div class="header-content">Chrome</div>
+				<main>Real main</main>
+			</body></html>
+		`);
+
+		expect(result.main?.nodeName).toBe('MAIN');
+		expect(result.wordCount).toBe(8);
+	});
+
+	it('may match class*=main on maintenance via Phase-2 (accepted false positive)', () => {
+		const result = extract(
+			'<html><body><div class="maintenance">Down</div></body></html>',
+		);
+
+		expect(result.main?.classList).toContain('maintenance');
+		expect(result.wordCount).toBe(4);
+	});
+
+	it('builds a simple diagnostic selector from tag, id, and classes', () => {
+		const result = extract(
+			'<html><body><div id="page-body" class="foo bar" role="main">X</div></body></html>',
+		);
+
+		expect(result.main?.selector).toBe('div#page-body.foo.bar');
+	});
+});
+
+describe('getMainContents', () => {
+	it('passes only the selector into page.evaluate so Document defaults in the page realm', async () => {
+		const evaluate = vi.fn(
+			(fn: typeof extractMainContentsFromDocument, sel: string | null) =>
+				Promise.resolve(
+					fn(sel, createDocument('<html><body><main>Hi</main></body></html>')),
+				),
+		);
+		const page = { evaluate } as unknown as Page;
+
+		const result = await getMainContents(page, { mainContentSelector: '#x' });
+
+		expect(evaluate).toHaveBeenCalledWith(extractMainContentsFromDocument, '#x');
+		expect(result.wordCount).toBe(2);
+	});
+});