@d-zero/beholder 3.1.6 → 4.0.0

package/src/get-main-contents.ts

@@ -0,0 +1,306 @@
+/**
+ * Main-content quantitative extraction for scraped HTML pages.
+ *
+ * Runs entirely against a `Document` (Puppeteer page realm or jsdom) so the
+ * same logic is unit-tested without a browser. Puppeteer callers use
+ * {@link getMainContents}, which passes this function into `page.evaluate`.
+ *
+ * WHY no `@medv/finder`: selector strings are diagnostic only; a tag+id+class // cspell:disable-line
+ * path is enough and avoids a Node-only dependency inside the page realm.
+ * @module
+ */
+
+import type { MainContentsData } from './types.js';
+import type { Page } from 'puppeteer';
+
+/**
+ * Extract main-content metrics from a `Document`.
+ *
+ * This function must remain free of closure over imports so Puppeteer can
+ * serialize it into `page.evaluate`.
+ *
+ * Argument order matches `page.evaluate(fn, mainContentSelector)`: the selector
+ * is the first argument; `doc` defaults to the page-realm global `document`
+ * when omitted (jsdom tests pass an explicit `Document` as the second argument).
+ * @param mainContentSelector - Optional selector prepended to the default list.
+ * @param doc - The document to inspect (defaults to the global `document`).
+ * @returns Quantitative main-content data (never `null`; empty when no main region).
+ * @example
+ * ```ts
+ * // In page.evaluate / browser:
+ * const data = extractMainContentsFromDocument('#page-body');
+ * // In jsdom tests:
+ * const data = extractMainContentsFromDocument('#page-body', document);
+ * ```
+ */
+export function extractMainContentsFromDocument(
+	mainContentSelector: string | null = null,
+	doc: Document = document,
+): MainContentsData {
+	/**
+	 * @param text
+	 */
+	function removeSpaces(text: string | null): string | null {
+		return text?.trim().replaceAll(/\s+/g, '') || null;
+	}
+
+	/**
+	 * @param value
+	 */
+	function cssEscape(value: string): string {
+		if (typeof CSS !== 'undefined' && typeof CSS.escape === 'function') {
+			return CSS.escape(value);
+		}
+		return value.replaceAll(/([ !"#$%&'()*+,./:;<=>?@[\\\]^`{|}~])/g, '\\$1');
+	}
+
+	/**
+	 * @param el
+	 */
+	function buildSelector(el: Element): string {
+		const tag = el.nodeName.toLowerCase();
+		const id = el.id ? `#${cssEscape(el.id)}` : '';
+		const classes = [...el.classList].map((c) => `.${cssEscape(c)}`).join('');
+		return `${tag}${id}${classes}`;
+	}
+
+	/**
+	 * @param media
+	 */
+	function resolveMediaSrc(media: HTMLMediaElement): string {
+		if (media.currentSrc) {
+			return media.currentSrc;
+		}
+		const attrSrc = media.getAttribute('src');
+		if (attrSrc) {
+			try {
+				return new URL(attrSrc, doc.baseURI).href;
+			} catch {
+				return attrSrc;
+			}
+		}
+		const source = media.querySelector('source[src]');
+		const sourceSrc = source?.getAttribute('src');
+		if (sourceSrc) {
+			try {
+				return new URL(sourceSrc, doc.baseURI).href;
+			} catch {
+				return sourceSrc;
+			}
+		}
+		return '';
+	}
+
+	const title = doc.title?.trim() ?? '';
+	const bodyWordCount = removeSpaces(doc.body?.textContent ?? null)?.length ?? 0;
+
+	const selectors = [
+		'main',
+		'[role="main"]',
+		'#main',
+		'.main',
+		'#content',
+		'.content',
+		'#contents',
+		'.contents',
+		'#main-content',
+		'.main-content',
+		'#main_content',
+		'.main_content',
+		'#mainContent',
+		'.mainContent',
+	];
+	if (mainContentSelector) {
+		selectors.unshift(mainContentSelector);
+	}
+
+	let $main: Element | null = null;
+	try {
+		$main = doc.querySelector(selectors.join(','));
+	} catch {
+		// Invalid custom selector: retry without it so built-in selectors still run.
+		if (mainContentSelector) {
+			selectors.shift();
+			$main = doc.querySelector(selectors.join(','));
+		}
+	}
+
+	if (!$main) {
+		const fallbackSelectors = [
+			'[id*="main" i]',
+			'[class*="main" i]',
+			'[id*="content" i]',
+			'[class*="content" i]',
+		];
+		for (const sel of fallbackSelectors) {
+			const candidate = doc.querySelector(sel);
+			if (candidate && candidate !== doc.body && candidate !== doc.documentElement) {
+				$main = candidate;
+				break;
+			}
+		}
+	}
+
+	if (!$main) {
+		return {
+			title,
+			main: null,
+			wordCount: 0,
+			bodyWordCount,
+			headings: [],
+			images: [],
+			tables: [],
+			buttons: [],
+			iframes: [],
+			videos: [],
+			audios: [],
+			canvases: [],
+		};
+	}
+
+	const headings: MainContentsData['headings'] = [];
+	for (const $heading of $main.querySelectorAll<HTMLHeadingElement>(
+		'h1, h2, h3, h4, h5, h6',
+	)) {
+		headings.push({
+			text: removeSpaces($heading.textContent),
+			level: Number.parseInt($heading.nodeName.replace(/h/i, ''), 10) as
+				1 | 2 | 3 | 4 | 5 | 6,
+		});
+	}
+
+	const images: MainContentsData['images'] = [];
+	for (const $img of $main.querySelectorAll<HTMLImageElement>(
+		'img, input[type="image"]',
+	)) {
+		images.push({
+			src: $img.src,
+			alt: $img.alt,
+		});
+	}
+
+	const tables: MainContentsData['tables'] = [];
+	for (const $table of $main.querySelectorAll<HTMLTableElement>('table')) {
+		tables.push({
+			rows: $table.querySelectorAll('tr').length,
+			cols: $table.querySelector('tr')?.querySelectorAll('th, td').length || 0,
+			hasHeader: !!$table.querySelector('thead'),
+			hasFooter: !!$table.querySelector('tfoot'),
+			hasMergedCell: !!$table.querySelector('[colspan], [rowspan]'),
+		});
+	}
+
+	const win = doc.defaultView;
+	const HTMLButton = win?.HTMLButtonElement;
+	const HTMLInput = win?.HTMLInputElement;
+
+	const buttons: MainContentsData['buttons'] = [];
+	for (const $el of $main.querySelectorAll(
+		'button, [role="button"], [class*="button"], [class*="btn"]',
+	)) {
+		const isButton = HTMLButton ? $el instanceof HTMLButton : $el.nodeName === 'BUTTON';
+		const isInput = HTMLInput ? $el instanceof HTMLInput : $el.nodeName === 'INPUT';
+		const isFormControl = isButton || isInput;
+		let text = removeSpaces($el.textContent);
+		if (!text && isInput) {
+			text = removeSpaces($el.getAttribute('value'));
+		}
+		const disabledProp =
+			isFormControl && 'disabled' in $el
+				? Boolean(($el as HTMLButtonElement).disabled)
+				: false;
+		buttons.push({
+			nodeName: $el.nodeName,
+			role: $el.getAttribute('role'),
+			type: isFormControl ? ($el as HTMLButtonElement | HTMLInputElement).type : null,
+			text,
+			disabled: disabledProp || $el.getAttribute('aria-disabled') === 'true',
+		});
+	}
+
+	const iframes: MainContentsData['iframes'] = [];
+	for (const $el of $main.querySelectorAll<HTMLIFrameElement>('iframe')) {
+		iframes.push({
+			src: $el.src,
+			title: $el.hasAttribute('title') ? $el.getAttribute('title') : null,
+			width: $el.getAttribute('width'),
+			height: $el.getAttribute('height'),
+		});
+	}
+
+	const videos: MainContentsData['videos'] = [];
+	for (const $el of $main.querySelectorAll<HTMLVideoElement>('video')) {
+		const posterAttr = $el.getAttribute('poster');
+		let poster: string | null = null;
+		if (posterAttr) {
+			try {
+				poster = new URL(posterAttr, doc.baseURI).href;
+			} catch {
+				poster = posterAttr;
+			}
+		}
+		videos.push({
+			src: resolveMediaSrc($el),
+			poster,
+			width: $el.width,
+			height: $el.height,
+		});
+	}
+
+	const audios: MainContentsData['audios'] = [];
+	for (const $el of $main.querySelectorAll<HTMLAudioElement>('audio')) {
+		audios.push({
+			src: resolveMediaSrc($el),
+		});
+	}
+
+	const canvases: MainContentsData['canvases'] = [];
+	for (const $el of $main.querySelectorAll<HTMLCanvasElement>('canvas')) {
+		canvases.push({
+			width: $el.width,
+			height: $el.height,
+		});
+	}
+
+	return {
+		title,
+		main: {
+			nodeName: $main.nodeName,
+			id: $main.id || null,
+			classList: [...$main.classList],
+			role: $main.getAttribute('role'),
+			selector: buildSelector($main),
+		},
+		wordCount: removeSpaces($main.textContent)?.length ?? 0,
+		bodyWordCount,
+		headings,
+		images,
+		tables,
+		buttons,
+		iframes,
+		videos,
+		audios,
+		canvases,
+	};
+}
+
+/**
+ * Extract main-content metrics from a Puppeteer page via a single `page.evaluate`.
+ * @param page - Puppeteer page whose DOM has finished loading.
+ * @param options - Optional main-content selector override.
+ * @param options.mainContentSelector
+ * @returns Quantitative main-content data.
+ * @example
+ * ```ts
+ * const mainContents = await getMainContents(page, { mainContentSelector: '#page-body' });
+ * ```
+ */
+export async function getMainContents(
+	page: Page,
+	options?: { mainContentSelector?: string | null },
+): Promise<MainContentsData> {
+	return page.evaluate(
+		extractMainContentsFromDocument,
+		options?.mainContentSelector ?? null,
+	);
+}

package/src/index.ts

@@ -27,6 +27,17 @@ export type {
 	ImageElement,
 	SkippedPageData,
 	NetworkLog,
+	ScrollHeightData,
+	MainContentsData,
+	MainContentsMainTag,
+	MainContentsHeading,
+	MainContentsImage,
+	MainContentsTable,
+	MainContentsButton,
+	MainContentsIframe,
+	MainContentsVideo,
+	MainContentsAudio,
+	MainContentsCanvas,
 	OpenGraphMeta,
 	OgArticleMeta,
 	OgBookMeta,

package/src/is-html-content-type.spec.ts

@@ -0,0 +1,17 @@
+import { describe, expect, it } from 'vitest';
+
+import { isHtmlContentType } from './is-html-content-type.js';
+
+describe('isHtmlContentType', () => {
+	it('accepts text/html in any letter case and with surrounding whitespace', () => {
+		expect(isHtmlContentType('text/html')).toBe(true);
+		expect(isHtmlContentType('Text/HTML')).toBe(true);
+		expect(isHtmlContentType(' text/html ')).toBe(true);
+	});
+
+	it('rejects non-HTML and null', () => {
+		expect(isHtmlContentType('application/pdf')).toBe(false);
+		expect(isHtmlContentType('text/plain')).toBe(false);
+		expect(isHtmlContentType(null)).toBe(false);
+	});
+});

package/src/is-html-content-type.ts

@@ -0,0 +1,20 @@
+/**
+ * Determine whether a Content-Type media type is HTML.
+ *
+ * MIME types are case-insensitive (RFC 2045). Surrounding whitespace after
+ * parameter stripping is also tolerated so `text/html `, `Text/HTML`, and
+ * `text/html; charset=utf-8` (after `;` split) all classify as HTML.
+ * @param contentType - The media type portion of a Content-Type header
+ *   (parameters already stripped), or `null` when unknown.
+ * @returns `true` when the media type is `text/html` in any letter case.
+ * @example
+ * ```ts
+ * isHtmlContentType('text/html'); // true
+ * isHtmlContentType('Text/HTML'); // true
+ * isHtmlContentType('application/pdf'); // false
+ * // Callers should strip parameters first (e.g. split on ';') before calling.
+ * ```
+ */
+export function isHtmlContentType(contentType: string | null): boolean {
+	return contentType !== null && contentType.trim().toLowerCase() === 'text/html';
+}

package/src/measure-scroll-height.spec.ts

@@ -0,0 +1,47 @@
+import type { Page } from 'puppeteer';
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { measureScrollHeight } from './measure-scroll-height.js';
+
+describe('measureScrollHeight', () => {
+	let page: Page;
+	let scrollHeights: number[];
+
+	beforeEach(() => {
+		scrollHeights = [2400, 5200];
+		let call = 0;
+		page = {
+			setViewport: vi.fn(() => Promise.resolve()),
+			evaluate: vi.fn(() => Promise.resolve(scrollHeights[call++] ?? 0)),
+		} as unknown as Page;
+	});
+
+	it('returns desktop and mobile heights without scrolling the page', async () => {
+		const result = await measureScrollHeight(page);
+
+		expect(result).toEqual({ desktop: 2400, mobile: 5200 });
+		expect(page.setViewport).toHaveBeenCalledTimes(2);
+		expect(page.setViewport).toHaveBeenNthCalledWith(1, {
+			width: 1280,
+			height: 800,
+			deviceScaleFactor: 1,
+		});
+		expect(page.setViewport).toHaveBeenNthCalledWith(2, {
+			width: 320,
+			height: 800,
+			deviceScaleFactor: 2,
+		});
+	});
+
+	it('returns null for a preset that throws', async () => {
+		page.setViewport = vi
+			.fn()
+			.mockResolvedValueOnce()
+			.mockRejectedValueOnce(new Error('viewport failed'));
+
+		const result = await measureScrollHeight(page);
+
+		expect(result).toEqual({ desktop: 2400, mobile: null });
+	});
+});

package/src/measure-scroll-height.ts

@@ -0,0 +1,52 @@
+/**
+ * Lightweight scrollHeight measurement without `scrollAllOver`.
+ *
+ * Used when `captureImages` is false so callers still receive desktop/mobile
+ * heights without the cost of full-page scrolling for image extraction.
+ * @module
+ */
+
+import type { ScrollHeightData } from './types.js';
+import type { Page } from 'puppeteer';
+
+import { devicePresets } from '@d-zero/puppeteer-page-scan';
+
+/**
+ * Measure `document.body.scrollHeight` at desktop-compact and mobile-small viewports.
+ *
+ * Sets each viewport, reads height once, and does **not** call `scrollAllOver`.
+ * A failed preset yields `null` for that side only.
+ * @param page - Puppeteer page whose DOM has finished loading.
+ * @returns Heights for desktop and mobile presets.
+ * @example
+ * ```ts
+ * const scrollHeight = await measureScrollHeight(page);
+ * // { desktop: 2400, mobile: 5200 }
+ * ```
+ */
+export async function measureScrollHeight(page: Page): Promise<ScrollHeightData> {
+	const desktop = await measureAtPreset(page, 'desktop-compact');
+	const mobile = await measureAtPreset(page, 'mobile-small');
+	return { desktop, mobile };
+}
+
+/**
+ * @param page
+ * @param key
+ */
+async function measureAtPreset(
+	page: Page,
+	key: 'desktop-compact' | 'mobile-small',
+): Promise<number | null> {
+	const preset: { width: number; resolution?: number } = devicePresets[key];
+	try {
+		await page.setViewport({
+			width: preset.width,
+			height: 800,
+			deviceScaleFactor: preset.resolution ?? 1,
+		});
+		return await page.evaluate(() => document.body.scrollHeight);
+	} catch {
+		return null;
+	}
+}

package/src/scraper.ts

@@ -10,6 +10,7 @@ import type {
 	PageData,
 	ParseURLOptions,
 	Resource,
+	ScrollHeightData,
 	SkippedPageData,
 } from './types.js';
 import type { PageScanPhase } from '@d-zero/puppeteer-page-scan';
@@ -28,8 +29,11 @@ import {
 	getImageList,
 	getMeta,
 } from './dom-evaluation.js';
+import { getMainContents } from './get-main-contents.js';
 import { isError } from './is-error.js';
+import { isHtmlContentType } from './is-html-content-type.js';
 import { keywordCheck } from './keyword-check.js';
+import { measureScrollHeight } from './measure-scroll-height.js';
 import { emptyMeta } from './meta/classify.js';
 import { findDisconnectionFailures } from './network-disconnection.js';
 import { parseUrl } from './parse-url.js';
@@ -148,6 +152,8 @@ export default class Scraper extends EventEmitter<ScraperEventTypes> {
 				imageList: [],
 				anchorList: [],
 				html: '',
+				mainContents: null,
+				scrollHeight: null,
 				isSkipped: false,
 			};
 
@@ -176,12 +182,14 @@ export default class Scraper extends EventEmitter<ScraperEventTypes> {
 				pageData: {
 					...headResult,
 					isTarget: false,
+					mainContents: headResult.mainContents ?? null,
+					scrollHeight: headResult.scrollHeight ?? null,
 				},
 				resources,
 			};
 		}
 
-		if (headResult === null || headResult.contentType === 'text/html') {
+		if (headResult === null || isHtmlContentType(headResult.contentType)) {
 			const fetchResult = await this.#fetchData(
 				page,
 				url,
@@ -555,7 +563,7 @@ export default class Scraper extends EventEmitter<ScraperEventTypes> {
 		const _contentLength = Number.parseInt(responseHeaders['content-length'] ?? '');
 		const contentLength = Number.isFinite(_contentLength) ? _contentLength : null;
 
-		if (contentType !== 'text/html') {
+		if (!isHtmlContentType(contentType)) {
 			return {
 				url,
 				isTarget: false,
@@ -570,6 +578,8 @@ export default class Scraper extends EventEmitter<ScraperEventTypes> {
 				imageList: [],
 				anchorList: [],
 				html: '',
+				mainContents: null,
+				scrollHeight: null,
 				isSkipped: false,
 			};
 		}
@@ -619,6 +629,8 @@ export default class Scraper extends EventEmitter<ScraperEventTypes> {
 				imageList: [],
 				anchorList: [],
 				html,
+				mainContents: null,
+				scrollHeight: null,
 				isSkipped: false,
 			};
 		}
@@ -644,6 +656,10 @@ export default class Scraper extends EventEmitter<ScraperEventTypes> {
 			throw new Error(`Network disconnection detected during page load: ${errorSummary}`);
 		}
 
+		const mainContents = await getMainContents(page, {
+			mainContentSelector: options?.mainContentSelector,
+		});
+
 		void this.emit('changePhase', {
 			pid: process.pid,
 			name: 'getAnchors',
@@ -671,24 +687,29 @@ export default class Scraper extends EventEmitter<ScraperEventTypes> {
 			domEvaluationTimeout,
 		);
 
-		const imageList = captureImages
-			? await (async () => {
-					void this.emit('changePhase', {
-						pid: process.pid,
-						name: 'extractImages',
-						url,
-						isExternal,
-						message: `%countdown(${domEvaluationTimeout},extractImages_${url.withoutHash},s)%s`,
-					});
-					return this.#fetchImages(
-						page,
-						url.withoutHashAndAuth,
-						isExternal,
-						imageLoadTimeout,
-						domEvaluationTimeout,
-					);
-				})()
-			: [];
+		let imageList: ImageElement[] = [];
+		let scrollHeight: ScrollHeightData | null = null;
+
+		if (captureImages) {
+			void this.emit('changePhase', {
+				pid: process.pid,
+				name: 'extractImages',
+				url,
+				isExternal,
+				message: `%countdown(${domEvaluationTimeout},extractImages_${url.withoutHash},s)%s`,
+			});
+			const fetched = await this.#fetchImages(
+				page,
+				url.withoutHashAndAuth,
+				isExternal,
+				imageLoadTimeout,
+				domEvaluationTimeout,
+			);
+			imageList = fetched.imageList;
+			scrollHeight = fetched.scrollHeight;
+		} else {
+			scrollHeight = await measureScrollHeight(page);
+		}
 
 		return {
 			url,
@@ -704,6 +725,8 @@ export default class Scraper extends EventEmitter<ScraperEventTypes> {
 			anchorList,
 			imageList,
 			html,
+			mainContents,
+			scrollHeight,
 			isSkipped: false,
 		};
 	}
@@ -719,12 +742,12 @@ export default class Scraper extends EventEmitter<ScraperEventTypes> {
 	 * changes and triggers a reload. Isolating each device preset allows partial
 	 * results — if one viewport fails, the other can still succeed.
 	 *
-	 * WHY retryable with 20-min timeout and `fallback: []`: Image extraction is
-	 * best-effort. If all retries fail, an empty array is returned rather than
-	 * failing the entire page scrape. The 20-min wall clock accommodates pages
-	 * whose mobile-small `scrollHeight` reaches ~300k px (observed on
-	 * responsive data tables, which take ~5 min to scroll). A shorter timeout
-	 * causes a second retry to start while the previous attempt's
+	 * WHY retryable with 20-min timeout and empty fallback: Image extraction is
+	 * best-effort. If all retries fail, empty images and null scroll heights are
+	 * returned rather than failing the entire page scrape. The 20-min wall clock
+	 * accommodates pages whose mobile-small `scrollHeight` reaches ~300k px
+	 * (observed on responsive data tables, which take ~5 min to scroll). A shorter
+	 * timeout causes a second retry to start while the previous attempt's
 	 * `scrollAllOver` is still running its `page.evaluate` calls in the
 	 * background — `Promise.race` in `retry.ts` does not cancel `fn()`. The
 	 * collision then surfaces as "Attempted to use detached Frame" or
@@ -736,16 +759,22 @@ export default class Scraper extends EventEmitter<ScraperEventTypes> {
 	 * preset entirely keeps the timeout-vs-background-evaluate collision from
 	 * ever being triggered, at the cost of losing that viewport's image data
 	 * for those pages. See {@link MAX_SCROLL_HEIGHT} for the chosen threshold.
+	 *
+	 * Scroll heights are still recorded when scrolling is skipped for the
+	 * height limit — the measurement from `beforePageScan` is kept.
 	 * @param page - Puppeteer page instance
 	 * @param url - The page URL string (without hash and auth)
 	 * @param isExternal - Whether the page is external
 	 * @param imageLoadTimeout - Timeout (ms) for waiting images to complete loading
 	 * @param domEvaluationTimeout - Timeout (ms) for the in-page image extraction `page.evaluate`
-	 * @returns Array of image elements from all device presets (may be partial if some viewports failed)
+	 * @returns Image elements plus desktop/mobile scroll heights from the scan path
 	 */
 	@retryable({
 		timeout: 20 * 60 * 1000,
-		fallback: [],
+		fallback: {
+			imageList: [],
+			scrollHeight: { desktop: null, mobile: null },
+		},
 		onWait(this: Scraper, determinedInterval, retryCount, methodName, error) {
 			void this.emit('changePhase', {
 				pid: process.pid,
@@ -771,15 +800,20 @@ export default class Scraper extends EventEmitter<ScraperEventTypes> {
 		isExternal: boolean,
 		imageLoadTimeout: number,
 		domEvaluationTimeout: number,
-	): Promise<ImageElement[]> {
+	): Promise<{ imageList: ImageElement[]; scrollHeight: ScrollHeightData }> {
 		const listener = this.#createPageScanListener(isExternal);
-		const devices: { key: string; preset: { width: number; resolution?: number } }[] = [
+		const devices: {
+			key: 'desktop-compact' | 'mobile-small';
+			preset: { width: number; resolution?: number };
+		}[] = [
 			{ key: 'desktop-compact', preset: devicePresets['desktop-compact'] },
 			{ key: 'mobile-small', preset: devicePresets['mobile-small'] },
 		];
 		const imageList: ImageElement[] = [];
+		const scrollHeight: ScrollHeightData = { desktop: null, mobile: null };
 
 		for (const { key, preset } of devices) {
+			const scrollKey = key === 'desktop-compact' ? 'desktop' : 'mobile';
 			try {
 				void this.emit('changePhase', {
 					pid: process.pid,
@@ -798,6 +832,8 @@ export default class Scraper extends EventEmitter<ScraperEventTypes> {
 					maxScrollHeight: MAX_SCROLL_HEIGHT,
 				});
 
+				scrollHeight[scrollKey] = scanResult.scrollHeight;
+
 				if (!scanResult.scrolled) {
 					void this.emit('changePhase', {
 						pid: process.pid,
@@ -845,6 +881,6 @@ export default class Scraper extends EventEmitter<ScraperEventTypes> {
 			}
 		}
 
-		return imageList;
+		return { imageList, scrollHeight };
 	}
 }