@d-zero/beholder 3.1.6 → 4.1.0

package/src/to-console-log-entry.spec.ts

@@ -0,0 +1,108 @@
+import type { ConsoleMessage, ConsoleMessageLocation, JSHandle } from 'puppeteer';
+
+import { describe, expect, it, vi } from 'vitest';
+
+import { toConsoleLogEntry } from './to-console-log-entry.js';
+
+/**
+ * Builds a minimal `JSHandle` mock whose `jsonValue()` resolves with the given value.
+ * @param value
+ */
+function mockArg(value: unknown): JSHandle {
+	return {
+		jsonValue: () => Promise.resolve(value),
+		dispose: () => Promise.resolve(),
+	} as unknown as JSHandle;
+}
+
+/**
+ * Builds a `JSHandle` mock whose `jsonValue()` rejects, simulating a handle
+ * that can no longer be resolved (e.g. a destroyed execution context).
+ * @param reason
+ */
+function mockRejectingArg(reason: unknown): JSHandle {
+	return {
+		jsonValue: () => Promise.reject(reason),
+		dispose: () => Promise.resolve(),
+	} as unknown as JSHandle;
+}
+
+/**
+ * Builds a minimal `ConsoleMessage` mock.
+ * @param options
+ * @param options.type
+ * @param options.text
+ * @param options.args
+ * @param options.location
+ */
+function mockConsoleMessage(options: {
+	type?: string;
+	text?: string;
+	args?: JSHandle[];
+	location?: ConsoleMessageLocation;
+}): ConsoleMessage {
+	return {
+		type: () => options.type ?? 'log',
+		text: () => options.text ?? '',
+		args: () => options.args ?? [],
+		location: () => options.location ?? {},
+	} as unknown as ConsoleMessage;
+}
+
+describe('toConsoleLogEntry', () => {
+	it('resolves text, type, and pageUrl', async () => {
+		const msg = mockConsoleMessage({ type: 'warn', text: 'careful' });
+		const entry = await toConsoleLogEntry(msg, 'https://example.com/');
+
+		expect(entry.pageUrl).toBe('https://example.com/');
+		expect(entry.type).toBe('warn');
+		expect(entry.text).toBe('careful');
+		expect(entry.args).toEqual([]);
+	});
+
+	it('resolves each argument via jsonValue()', async () => {
+		const msg = mockConsoleMessage({
+			args: [mockArg('a'), mockArg(1), mockArg({ b: 2 })],
+		});
+		const entry = await toConsoleLogEntry(msg, 'https://example.com/');
+
+		expect(entry.args).toEqual(['a', 1, { b: 2 }]);
+	});
+
+	it('falls back to undefined for an argument whose jsonValue() rejects', async () => {
+		const msg = mockConsoleMessage({
+			args: [mockArg('ok'), mockRejectingArg(new Error('destroyed context'))],
+		});
+		const entry = await toConsoleLogEntry(msg, 'https://example.com/');
+
+		expect(entry.args).toEqual(['ok', undefined]);
+	});
+
+	it('disposes each argument handle after resolving', async () => {
+		const arg = mockArg('a');
+		const disposeSpy = vi.spyOn(arg, 'dispose');
+		const msg = mockConsoleMessage({ args: [arg] });
+		await toConsoleLogEntry(msg, 'https://example.com/');
+
+		expect(disposeSpy).toHaveBeenCalledOnce();
+	});
+
+	it('keeps location when the message has a source URL', async () => {
+		const location = {
+			url: 'https://example.com/app.js',
+			lineNumber: 3,
+			columnNumber: 7,
+		};
+		const msg = mockConsoleMessage({ location });
+		const entry = await toConsoleLogEntry(msg, 'https://example.com/');
+
+		expect(entry.location).toEqual(location);
+	});
+
+	it('omits location when the message has no source URL', async () => {
+		const msg = mockConsoleMessage({ location: {} });
+		const entry = await toConsoleLogEntry(msg, 'https://example.com/');
+
+		expect(entry.location).toBeUndefined();
+	});
+});

package/src/to-console-log-entry.ts

@@ -0,0 +1,40 @@
+import type { ConsoleLogEntry } from './types.js';
+import type { ConsoleMessage } from 'puppeteer';
+
+/**
+ * Converts a Puppeteer `ConsoleMessage` into a `ConsoleLogEntry`.
+ *
+ * WHY per-argument try/catch: `JSHandle.jsonValue()` can reject when the
+ * page's execution context has been destroyed (e.g. mid-navigation) or when
+ * the value cannot be serialized (e.g. a circular reference). A single
+ * failing argument must not drop the rest of the message's arguments.
+ * @param msg - The console message captured via `page.on('console')`
+ * @param pageUrl - The URL (without hash) of the page that produced the message
+ * @returns The resolved console log entry
+ */
+export async function toConsoleLogEntry(
+	msg: ConsoleMessage,
+	pageUrl: string,
+): Promise<ConsoleLogEntry> {
+	const args = await Promise.all(
+		msg.args().map(async (arg) => {
+			try {
+				return await arg.jsonValue();
+			} catch {
+				return;
+			} finally {
+				await arg.dispose().catch(() => {});
+			}
+		}),
+	);
+	const loc = msg.location();
+
+	return {
+		pageUrl,
+		type: msg.type(),
+		text: msg.text(),
+		args,
+		location: loc.url === undefined ? undefined : loc,
+		ts: Date.now(),
+	};
+}

package/src/to-page-error-entry.spec.ts

@@ -0,0 +1,23 @@
+import { describe, expect, it } from 'vitest';
+
+import { toPageErrorEntry } from './to-page-error-entry.js';
+
+describe('toPageErrorEntry', () => {
+	it('extracts message and stack from an Error', () => {
+		const error = new Error('boom');
+		const entry = toPageErrorEntry(error, 'https://example.com/');
+
+		expect(entry.pageUrl).toBe('https://example.com/');
+		expect(entry.type).toBe('pageerror');
+		expect(entry.text).toBe('boom');
+		expect(entry.stack).toBe(error.stack);
+		expect(entry.args).toEqual([]);
+	});
+
+	it('stringifies a non-Error throw value and omits stack', () => {
+		const entry = toPageErrorEntry('oops', 'https://example.com/');
+
+		expect(entry.text).toBe('oops');
+		expect(entry.stack).toBeUndefined();
+	});
+});

package/src/to-page-error-entry.ts

@@ -0,0 +1,22 @@
+import type { ConsoleLogEntry } from './types.js';
+
+/**
+ * Converts an uncaught exception / unhandled Promise rejection (captured via
+ * `page.on('pageerror')`) into a `ConsoleLogEntry`.
+ * @param error - The value emitted by `page.on('pageerror')`; typed as `unknown`
+ * because Puppeteer allows non-`Error` throw values (e.g. `throw 'oops'`)
+ * @param pageUrl - The URL (without hash) of the page that produced the error
+ * @returns The console log entry, with `type: 'pageerror'`
+ */
+export function toPageErrorEntry(error: unknown, pageUrl: string): ConsoleLogEntry {
+	const isErr = error instanceof Error;
+
+	return {
+		pageUrl,
+		type: 'pageerror',
+		text: isErr ? error.message : String(error),
+		args: [],
+		stack: isErr ? error.stack : undefined,
+		ts: Date.now(),
+	};
+}

package/src/types.ts

@@ -59,6 +59,7 @@ import type { Meta } from './meta/types.js';
 import type { CDNType } from '@d-zero/shared/detect-cdn';
 import type { CompressType } from '@d-zero/shared/detect-compress';
 import type { ExURL } from '@d-zero/shared/parse-url';
+import type { ConsoleMessageType } from 'puppeteer';
 
 /**
  * Scraped page data returned by the scraper after successfully processing a page.
@@ -103,10 +104,201 @@ export type PageData = {
 	/** HTML snapshot of the rendered DOM. */
 	html: string;
 
+	/**
+	 * Quantitative main-content metrics extracted in the browser after load.
+	 * `null` when the page is non-HTML, external, non-HTTP, or otherwise not fully scraped.
+	 * When HTML is scraped but no main region is found, this is still an object with empty arrays and `wordCount: 0`.
+	 */
+	mainContents: MainContentsData | null;
+
+	/**
+	 * `document.body.scrollHeight` at desktop-compact and mobile-small viewports.
+	 * `null` when not measured (non-HTML, external, non-HTTP, or skipped).
+	 */
+	scrollHeight: ScrollHeightData | null;
+
 	/** Always `false` for successfully scraped pages. See {@link SkippedPageData} for skipped pages. */
 	isSkipped: false;
 };
 
+/**
+ * `document.body.scrollHeight` measured at the scraper's desktop and mobile device presets.
+ */
+export type ScrollHeightData = {
+	/** Height at `desktop-compact` (width 1280), or `null` if that preset failed. */
+	desktop: number | null;
+	/** Height at `mobile-small` (width 320 @ 2x), or `null` if that preset failed. */
+	mobile: number | null;
+};
+
+/**
+ * Quantitative metrics for the detected main content region of a page.
+ * @example
+ * ```ts
+ * const { mainContents } = pageData;
+ * if (mainContents) {
+ *   console.log(mainContents.wordCount, mainContents.headings.length);
+ * }
+ * ```
+ */
+export type MainContentsData = {
+	/** `document.title` trimmed. Empty string when the title is empty. */
+	title: string;
+	/**
+	 * Identifying info for the detected main region element.
+	 * `null` when no selector matched (arrays are empty and `wordCount` is `0`).
+	 */
+	main: MainContentsMainTag | null;
+	/**
+	 * Character count of the main region's `textContent` after whitespace removal.
+	 * `0` when no main region was found or the text is whitespace-only.
+	 */
+	wordCount: number;
+	/**
+	 * Character count of `document.body` `textContent` after whitespace removal.
+	 * Measured even when no main region is found (`0` if `body` is missing).
+	 */
+	bodyWordCount: number;
+	/** Headings (`h1`–`h6`) inside the main region, in DOM order. */
+	headings: MainContentsHeading[];
+	/** Images (`img`, `input[type=image]`) inside the main region, in DOM order. */
+	images: MainContentsImage[];
+	/** Tables inside the main region, in DOM order. */
+	tables: MainContentsTable[];
+	/** Button-like elements inside the main region, in DOM order. */
+	buttons: MainContentsButton[];
+	/** Iframes inside the main region, in DOM order. */
+	iframes: MainContentsIframe[];
+	/** Videos inside the main region, in DOM order. */
+	videos: MainContentsVideo[];
+	/** Audios inside the main region, in DOM order. */
+	audios: MainContentsAudio[];
+	/** Canvases inside the main region, in DOM order. */
+	canvases: MainContentsCanvas[];
+};
+
+/**
+ * Identifying information about the DOM element used as the main content area.
+ */
+export type MainContentsMainTag = {
+	/** Element tag name (e.g. `"MAIN"`, `"DIV"`), as reported by `nodeName`. */
+	nodeName: string;
+	/** The element's `id`, or `null` when empty/absent. */
+	id: string | null;
+	/** CSS class names on the element. */
+	classList: string[];
+	/** The WAI-ARIA `role` attribute value, or `null` when absent. */
+	role: string | null;
+	/**
+	 * A simple diagnostic CSS selector built from tag + id + classes
+	 * (not `@medv/finder`). // cspell:disable-line
+	 */
+	selector: string;
+};
+
+/**
+ * A heading element (`h1`–`h6`) found within the main content area.
+ */
+export type MainContentsHeading = {
+	/** Heading text after whitespace removal, or `null` when empty. */
+	text: string | null;
+	/** Heading level from the tag name. */
+	level: 1 | 2 | 3 | 4 | 5 | 6;
+};
+
+/**
+ * An `<img>` or `<input type="image">` element found in the main content.
+ */
+export type MainContentsImage = {
+	/** Resolved absolute `src` URL. */
+	src: string;
+	/** `alt` attribute value (may be an empty string). */
+	alt: string;
+};
+
+/**
+ * Structural summary of a `<table>` found within the main content.
+ */
+export type MainContentsTable = {
+	/** Number of `<tr>` elements. */
+	rows: number;
+	/** Number of `th`/`td` cells in the first row, or `0` when there is no row. */
+	cols: number;
+	/** Whether the table contains a `<thead>`. */
+	hasHeader: boolean;
+	/** Whether the table contains a `<tfoot>`. */
+	hasFooter: boolean;
+	/** Whether any cell uses `colspan` or `rowspan`. */
+	hasMergedCell: boolean;
+};
+
+/**
+ * A button-like element found in the main content
+ * (`button`, `[role=button]`, `[class*=button]`, `[class*=btn]`).
+ */
+export type MainContentsButton = {
+	/** Element tag name (e.g. `"BUTTON"`, `"A"`, `"DIV"`). */
+	nodeName: string;
+	/** `role` attribute, or `null` when absent. */
+	role: string | null;
+	/** `type` for `<button>` / `<input>`, otherwise `null`. */
+	type: string | null;
+	/** Label text after whitespace removal, or `null` when empty. */
+	text: string | null;
+	/** `true` when `disabled` or `aria-disabled="true"`. */
+	disabled: boolean;
+};
+
+/**
+ * An `<iframe>` found in the main content.
+ */
+export type MainContentsIframe = {
+	/** Resolved absolute `src` URL (empty string when unset). */
+	src: string;
+	/** `title` attribute, or `null` when the attribute is absent. */
+	title: string | null;
+	/** Raw `width` attribute string, or `null` when absent. */
+	width: string | null;
+	/** Raw `height` attribute string, or `null` when absent. */
+	height: string | null;
+};
+
+/**
+ * A `<video>` found in the main content.
+ */
+export type MainContentsVideo = {
+	/**
+	 * Resolved media URL: `currentSrc`, else `src` attribute, else first `source[src]`, else `""`.
+	 */
+	src: string;
+	/** Resolved `poster` URL, or `null` when unset. */
+	poster: string | null;
+	/** IDL `width` in pixels. */
+	width: number;
+	/** IDL `height` in pixels. */
+	height: number;
+};
+
+/**
+ * An `<audio>` found in the main content.
+ */
+export type MainContentsAudio = {
+	/**
+	 * Resolved media URL: `currentSrc`, else `src` attribute, else first `source[src]`, else `""`.
+	 */
+	src: string;
+};
+
+/**
+ * A `<canvas>` found in the main content.
+ */
+export type MainContentsCanvas = {
+	/** IDL bitmap width. */
+	width: number;
+	/** IDL bitmap height. */
+	height: number;
+};
+
 /**
  * Information about an image element found on a page.
  */
@@ -286,6 +478,8 @@ export type ScrapeResult = {
 	pageData?: PageData;
 	/** All sub-resources captured during the page load. */
 	resources: ResourceEntry[];
+	/** All console messages and uncaught page errors captured during the page load. */
+	consoleLogs: ConsoleLogEntry[];
 	/** Details about why the page was ignored, present when `type` is `"skipped"`. */
 	ignored?: { url: ExURL; matchedText: string; excludeKeywords: string[] };
 	/** Error details, present when `type` is `"error"`. */
@@ -307,6 +501,35 @@ export type ResourceEntry = {
 	pageUrl: string;
 };
 
+/**
+ * A single console message or uncaught page error captured during page scraping.
+ * Captured via Puppeteer's `console` and `pageerror` page events; internal pages only.
+ */
+export type ConsoleLogEntry = {
+	/** The URL (without hash) of the page that produced this message. */
+	pageUrl: string;
+	/**
+	 * The console message type (Puppeteer's `ConsoleMessageType`, e.g. `"log"`, `"warn"`, `"error"`),
+	 * or `"pageerror"` for an uncaught exception / unhandled Promise rejection.
+	 */
+	type: ConsoleMessageType | 'pageerror';
+	/** The message text, or the error's `message` for `"pageerror"` entries. */
+	text: string;
+	/**
+	 * Arguments passed to the console call, resolved via `JSHandle.jsonValue()`.
+	 * An individual argument is `undefined` when it could not be resolved (e.g. a
+	 * destroyed execution context or a value `jsonValue()` cannot serialize).
+	 * Always empty for `"pageerror"` entries.
+	 */
+	args: unknown[];
+	/** Source location of the message, or `undefined` when unavailable (always `undefined` for `"pageerror"`). */
+	location?: { url?: string; lineNumber?: number; columnNumber?: number };
+	/** Stack trace text, present only for `"pageerror"` entries. */
+	stack?: string;
+	/** Timestamp (ms since epoch) when the message was captured. */
+	ts: number;
+};
+
 /**
  * Event payload describing a phase transition in the scraping lifecycle.
  * Phases proceed roughly in order: scrapeStart -> openPage ->
@@ -427,4 +650,9 @@ export type ScraperOptions = {
 	 * Default: 180_000 (180s, aligned with the upstream retryable timeout).
 	 */
 	domEvaluationTimeout?: number;
+	/**
+	 * CSS selector prepended to the default main-content detection list
+	 * (highest priority). When omitted or `null`, only the built-in selectors are used.
+	 */
+	mainContentSelector?: string | null;
 };