@d-zero/beholder 3.1.6 → 4.1.0

package/src/get-main-contents.ts

@@ -0,0 +1,306 @@
+/**
+ * Main-content quantitative extraction for scraped HTML pages.
+ *
+ * Runs entirely against a `Document` (Puppeteer page realm or jsdom) so the
+ * same logic is unit-tested without a browser. Puppeteer callers use
+ * {@link getMainContents}, which passes this function into `page.evaluate`.
+ *
+ * WHY no `@medv/finder`: selector strings are diagnostic only; a tag+id+class // cspell:disable-line
+ * path is enough and avoids a Node-only dependency inside the page realm.
+ * @module
+ */
+
+import type { MainContentsData } from './types.js';
+import type { Page } from 'puppeteer';
+
+/**
+ * Extract main-content metrics from a `Document`.
+ *
+ * This function must remain free of closure over imports so Puppeteer can
+ * serialize it into `page.evaluate`.
+ *
+ * Argument order matches `page.evaluate(fn, mainContentSelector)`: the selector
+ * is the first argument; `doc` defaults to the page-realm global `document`
+ * when omitted (jsdom tests pass an explicit `Document` as the second argument).
+ * @param mainContentSelector - Optional selector prepended to the default list.
+ * @param doc - The document to inspect (defaults to the global `document`).
+ * @returns Quantitative main-content data (never `null`; empty when no main region).
+ * @example
+ * ```ts
+ * // In page.evaluate / browser:
+ * const data = extractMainContentsFromDocument('#page-body');
+ * // In jsdom tests:
+ * const data = extractMainContentsFromDocument('#page-body', document);
+ * ```
+ */
+export function extractMainContentsFromDocument(
+	mainContentSelector: string | null = null,
+	doc: Document = document,
+): MainContentsData {
+	/**
+	 * @param text
+	 */
+	function removeSpaces(text: string | null): string | null {
+		return text?.trim().replaceAll(/\s+/g, '') || null;
+	}
+
+	/**
+	 * @param value
+	 */
+	function cssEscape(value: string): string {
+		if (typeof CSS !== 'undefined' && typeof CSS.escape === 'function') {
+			return CSS.escape(value);
+		}
+		return value.replaceAll(/([ !"#$%&'()*+,./:;<=>?@[\\\]^`{|}~])/g, '\\$1');
+	}
+
+	/**
+	 * @param el
+	 */
+	function buildSelector(el: Element): string {
+		const tag = el.nodeName.toLowerCase();
+		const id = el.id ? `#${cssEscape(el.id)}` : '';
+		const classes = [...el.classList].map((c) => `.${cssEscape(c)}`).join('');
+		return `${tag}${id}${classes}`;
+	}
+
+	/**
+	 * @param media
+	 */
+	function resolveMediaSrc(media: HTMLMediaElement): string {
+		if (media.currentSrc) {
+			return media.currentSrc;
+		}
+		const attrSrc = media.getAttribute('src');
+		if (attrSrc) {
+			try {
+				return new URL(attrSrc, doc.baseURI).href;
+			} catch {
+				return attrSrc;
+			}
+		}
+		const source = media.querySelector('source[src]');
+		const sourceSrc = source?.getAttribute('src');
+		if (sourceSrc) {
+			try {
+				return new URL(sourceSrc, doc.baseURI).href;
+			} catch {
+				return sourceSrc;
+			}
+		}
+		return '';
+	}
+
+	const title = doc.title?.trim() ?? '';
+	const bodyWordCount = removeSpaces(doc.body?.textContent ?? null)?.length ?? 0;
+
+	const selectors = [
+		'main',
+		'[role="main"]',
+		'#main',
+		'.main',
+		'#content',
+		'.content',
+		'#contents',
+		'.contents',
+		'#main-content',
+		'.main-content',
+		'#main_content',
+		'.main_content',
+		'#mainContent',
+		'.mainContent',
+	];
+	if (mainContentSelector) {
+		selectors.unshift(mainContentSelector);
+	}
+
+	let $main: Element | null = null;
+	try {
+		$main = doc.querySelector(selectors.join(','));
+	} catch {
+		// Invalid custom selector: retry without it so built-in selectors still run.
+		if (mainContentSelector) {
+			selectors.shift();
+			$main = doc.querySelector(selectors.join(','));
+		}
+	}
+
+	if (!$main) {
+		const fallbackSelectors = [
+			'[id*="main" i]',
+			'[class*="main" i]',
+			'[id*="content" i]',
+			'[class*="content" i]',
+		];
+		for (const sel of fallbackSelectors) {
+			const candidate = doc.querySelector(sel);
+			if (candidate && candidate !== doc.body && candidate !== doc.documentElement) {
+				$main = candidate;
+				break;
+			}
+		}
+	}
+
+	if (!$main) {
+		return {
+			title,
+			main: null,
+			wordCount: 0,
+			bodyWordCount,
+			headings: [],
+			images: [],
+			tables: [],
+			buttons: [],
+			iframes: [],
+			videos: [],
+			audios: [],
+			canvases: [],
+		};
+	}
+
+	const headings: MainContentsData['headings'] = [];
+	for (const $heading of $main.querySelectorAll<HTMLHeadingElement>(
+		'h1, h2, h3, h4, h5, h6',
+	)) {
+		headings.push({
+			text: removeSpaces($heading.textContent),
+			level: Number.parseInt($heading.nodeName.replace(/h/i, ''), 10) as
+				1 | 2 | 3 | 4 | 5 | 6,
+		});
+	}
+
+	const images: MainContentsData['images'] = [];
+	for (const $img of $main.querySelectorAll<HTMLImageElement>(
+		'img, input[type="image"]',
+	)) {
+		images.push({
+			src: $img.src,
+			alt: $img.alt,
+		});
+	}
+
+	const tables: MainContentsData['tables'] = [];
+	for (const $table of $main.querySelectorAll<HTMLTableElement>('table')) {
+		tables.push({
+			rows: $table.querySelectorAll('tr').length,
+			cols: $table.querySelector('tr')?.querySelectorAll('th, td').length || 0,
+			hasHeader: !!$table.querySelector('thead'),
+			hasFooter: !!$table.querySelector('tfoot'),
+			hasMergedCell: !!$table.querySelector('[colspan], [rowspan]'),
+		});
+	}
+
+	const win = doc.defaultView;
+	const HTMLButton = win?.HTMLButtonElement;
+	const HTMLInput = win?.HTMLInputElement;
+
+	const buttons: MainContentsData['buttons'] = [];
+	for (const $el of $main.querySelectorAll(
+		'button, [role="button"], [class*="button"], [class*="btn"]',
+	)) {
+		const isButton = HTMLButton ? $el instanceof HTMLButton : $el.nodeName === 'BUTTON';
+		const isInput = HTMLInput ? $el instanceof HTMLInput : $el.nodeName === 'INPUT';
+		const isFormControl = isButton || isInput;
+		let text = removeSpaces($el.textContent);
+		if (!text && isInput) {
+			text = removeSpaces($el.getAttribute('value'));
+		}
+		const disabledProp =
+			isFormControl && 'disabled' in $el
+				? Boolean(($el as HTMLButtonElement).disabled)
+				: false;
+		buttons.push({
+			nodeName: $el.nodeName,
+			role: $el.getAttribute('role'),
+			type: isFormControl ? ($el as HTMLButtonElement | HTMLInputElement).type : null,
+			text,
+			disabled: disabledProp || $el.getAttribute('aria-disabled') === 'true',
+		});
+	}
+
+	const iframes: MainContentsData['iframes'] = [];
+	for (const $el of $main.querySelectorAll<HTMLIFrameElement>('iframe')) {
+		iframes.push({
+			src: $el.src,
+			title: $el.hasAttribute('title') ? $el.getAttribute('title') : null,
+			width: $el.getAttribute('width'),
+			height: $el.getAttribute('height'),
+		});
+	}
+
+	const videos: MainContentsData['videos'] = [];
+	for (const $el of $main.querySelectorAll<HTMLVideoElement>('video')) {
+		const posterAttr = $el.getAttribute('poster');
+		let poster: string | null = null;
+		if (posterAttr) {
+			try {
+				poster = new URL(posterAttr, doc.baseURI).href;
+			} catch {
+				poster = posterAttr;
+			}
+		}
+		videos.push({
+			src: resolveMediaSrc($el),
+			poster,
+			width: $el.width,
+			height: $el.height,
+		});
+	}
+
+	const audios: MainContentsData['audios'] = [];
+	for (const $el of $main.querySelectorAll<HTMLAudioElement>('audio')) {
+		audios.push({
+			src: resolveMediaSrc($el),
+		});
+	}
+
+	const canvases: MainContentsData['canvases'] = [];
+	for (const $el of $main.querySelectorAll<HTMLCanvasElement>('canvas')) {
+		canvases.push({
+			width: $el.width,
+			height: $el.height,
+		});
+	}
+
+	return {
+		title,
+		main: {
+			nodeName: $main.nodeName,
+			id: $main.id || null,
+			classList: [...$main.classList],
+			role: $main.getAttribute('role'),
+			selector: buildSelector($main),
+		},
+		wordCount: removeSpaces($main.textContent)?.length ?? 0,
+		bodyWordCount,
+		headings,
+		images,
+		tables,
+		buttons,
+		iframes,
+		videos,
+		audios,
+		canvases,
+	};
+}
+
+/**
+ * Extract main-content metrics from a Puppeteer page via a single `page.evaluate`.
+ * @param page - Puppeteer page whose DOM has finished loading.
+ * @param options - Optional main-content selector override.
+ * @param options.mainContentSelector
+ * @returns Quantitative main-content data.
+ * @example
+ * ```ts
+ * const mainContents = await getMainContents(page, { mainContentSelector: '#page-body' });
+ * ```
+ */
+export async function getMainContents(
+	page: Page,
+	options?: { mainContentSelector?: string | null },
+): Promise<MainContentsData> {
+	return page.evaluate(
+		extractMainContentsFromDocument,
+		options?.mainContentSelector ?? null,
+	);
+}

package/src/index.ts

@@ -18,7 +18,7 @@ export { detectCompress } from '@d-zero/shared/detect-compress';
 export type { CompressType } from '@d-zero/shared/detect-compress';
 export { detectCDN } from '@d-zero/shared/detect-cdn';
 export type { CDNType } from '@d-zero/shared/detect-cdn';
-export type { ScrapeResult, ResourceEntry, PageData } from './types.js';
+export type { ScrapeResult, ResourceEntry, ConsoleLogEntry, PageData } from './types.js';
 export type { ScraperOptions, ChangePhaseEvent, ScraperEventTypes } from './types.js';
 export type {
 	Resource,
@@ -27,6 +27,17 @@ export type {
 	ImageElement,
 	SkippedPageData,
 	NetworkLog,
+	ScrollHeightData,
+	MainContentsData,
+	MainContentsMainTag,
+	MainContentsHeading,
+	MainContentsImage,
+	MainContentsTable,
+	MainContentsButton,
+	MainContentsIframe,
+	MainContentsVideo,
+	MainContentsAudio,
+	MainContentsCanvas,
 	OpenGraphMeta,
 	OgArticleMeta,
 	OgBookMeta,

package/src/is-html-content-type.spec.ts

@@ -0,0 +1,17 @@
+import { describe, expect, it } from 'vitest';
+
+import { isHtmlContentType } from './is-html-content-type.js';
+
+describe('isHtmlContentType', () => {
+	it('accepts text/html in any letter case and with surrounding whitespace', () => {
+		expect(isHtmlContentType('text/html')).toBe(true);
+		expect(isHtmlContentType('Text/HTML')).toBe(true);
+		expect(isHtmlContentType(' text/html ')).toBe(true);
+	});
+
+	it('rejects non-HTML and null', () => {
+		expect(isHtmlContentType('application/pdf')).toBe(false);
+		expect(isHtmlContentType('text/plain')).toBe(false);
+		expect(isHtmlContentType(null)).toBe(false);
+	});
+});

package/src/is-html-content-type.ts

@@ -0,0 +1,20 @@
+/**
+ * Determine whether a Content-Type media type is HTML.
+ *
+ * MIME types are case-insensitive (RFC 2045). Surrounding whitespace after
+ * parameter stripping is also tolerated so `text/html `, `Text/HTML`, and
+ * `text/html; charset=utf-8` (after `;` split) all classify as HTML.
+ * @param contentType - The media type portion of a Content-Type header
+ *   (parameters already stripped), or `null` when unknown.
+ * @returns `true` when the media type is `text/html` in any letter case.
+ * @example
+ * ```ts
+ * isHtmlContentType('text/html'); // true
+ * isHtmlContentType('Text/HTML'); // true
+ * isHtmlContentType('application/pdf'); // false
+ * // Callers should strip parameters first (e.g. split on ';') before calling.
+ * ```
+ */
+export function isHtmlContentType(contentType: string | null): boolean {
+	return contentType !== null && contentType.trim().toLowerCase() === 'text/html';
+}

package/src/measure-scroll-height.spec.ts

@@ -0,0 +1,47 @@
+import type { Page } from 'puppeteer';
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { measureScrollHeight } from './measure-scroll-height.js';
+
+describe('measureScrollHeight', () => {
+	let page: Page;
+	let scrollHeights: number[];
+
+	beforeEach(() => {
+		scrollHeights = [2400, 5200];
+		let call = 0;
+		page = {
+			setViewport: vi.fn(() => Promise.resolve()),
+			evaluate: vi.fn(() => Promise.resolve(scrollHeights[call++] ?? 0)),
+		} as unknown as Page;
+	});
+
+	it('returns desktop and mobile heights without scrolling the page', async () => {
+		const result = await measureScrollHeight(page);
+
+		expect(result).toEqual({ desktop: 2400, mobile: 5200 });
+		expect(page.setViewport).toHaveBeenCalledTimes(2);
+		expect(page.setViewport).toHaveBeenNthCalledWith(1, {
+			width: 1280,
+			height: 800,
+			deviceScaleFactor: 1,
+		});
+		expect(page.setViewport).toHaveBeenNthCalledWith(2, {
+			width: 320,
+			height: 800,
+			deviceScaleFactor: 2,
+		});
+	});
+
+	it('returns null for a preset that throws', async () => {
+		page.setViewport = vi
+			.fn()
+			.mockResolvedValueOnce()
+			.mockRejectedValueOnce(new Error('viewport failed'));
+
+		const result = await measureScrollHeight(page);
+
+		expect(result).toEqual({ desktop: 2400, mobile: null });
+	});
+});

package/src/measure-scroll-height.ts

@@ -0,0 +1,52 @@
+/**
+ * Lightweight scrollHeight measurement without `scrollAllOver`.
+ *
+ * Used when `captureImages` is false so callers still receive desktop/mobile
+ * heights without the cost of full-page scrolling for image extraction.
+ * @module
+ */
+
+import type { ScrollHeightData } from './types.js';
+import type { Page } from 'puppeteer';
+
+import { devicePresets } from '@d-zero/puppeteer-page-scan';
+
+/**
+ * Measure `document.body.scrollHeight` at desktop-compact and mobile-small viewports.
+ *
+ * Sets each viewport, reads height once, and does **not** call `scrollAllOver`.
+ * A failed preset yields `null` for that side only.
+ * @param page - Puppeteer page whose DOM has finished loading.
+ * @returns Heights for desktop and mobile presets.
+ * @example
+ * ```ts
+ * const scrollHeight = await measureScrollHeight(page);
+ * // { desktop: 2400, mobile: 5200 }
+ * ```
+ */
+export async function measureScrollHeight(page: Page): Promise<ScrollHeightData> {
+	const desktop = await measureAtPreset(page, 'desktop-compact');
+	const mobile = await measureAtPreset(page, 'mobile-small');
+	return { desktop, mobile };
+}
+
+/**
+ * @param page
+ * @param key
+ */
+async function measureAtPreset(
+	page: Page,
+	key: 'desktop-compact' | 'mobile-small',
+): Promise<number | null> {
+	const preset: { width: number; resolution?: number } = devicePresets[key];
+	try {
+		await page.setViewport({
+			width: preset.width,
+			height: 800,
+			deviceScaleFactor: preset.resolution ?? 1,
+		});
+		return await page.evaluate(() => document.body.scrollHeight);
+	} catch {
+		return null;
+	}
+}