@d-zero/beholder 3.1.5 → 4.0.0

package/src/get-main-contents.spec.ts

@@ -0,0 +1,282 @@
+import type { Page } from 'puppeteer';
+
+import { JSDOM } from 'jsdom';
+import { describe, expect, it, vi } from 'vitest';
+
+import { extractMainContentsFromDocument, getMainContents } from './get-main-contents.js';
+
+/**
+ * @param html
+ */
+function createDocument(html: string): Document {
+	return new JSDOM(html, { url: 'https://example.com/' }).window.document;
+}
+
+/**
+ * @param html
+ * @param mainContentSelector
+ */
+function extract(html: string, mainContentSelector: string | null = null) {
+	return extractMainContentsFromDocument(mainContentSelector, createDocument(html));
+}
+
+describe('extractMainContentsFromDocument', () => {
+	it('returns empty main metrics when no main content element is found, but counts body text', () => {
+		const result = extract(
+			'<html><head><title>T</title></head><body><header>ABC</header><div>No main</div></body></html>',
+		);
+
+		expect(result).toEqual({
+			title: 'T',
+			main: null,
+			wordCount: 0,
+			bodyWordCount: 9,
+			headings: [],
+			images: [],
+			tables: [],
+			buttons: [],
+			iframes: [],
+			videos: [],
+			audios: [],
+			canvases: [],
+		});
+	});
+
+	it('detects <main> and counts word length', () => {
+		const result = extract('<html><body><main>こんにちは世界</main></body></html>');
+
+		expect(result.wordCount).toBe(7);
+		expect(result.bodyWordCount).toBeGreaterThanOrEqual(7);
+		expect(result.main?.nodeName).toBe('MAIN');
+	});
+
+	it('detects [role="main"]', () => {
+		const result = extract('<html><body><div role="main">Content</div></body></html>');
+
+		expect(result.wordCount).toBe(7);
+		expect(result.main?.role).toBe('main');
+	});
+
+	it('uses custom mainContentSelector when provided', () => {
+		const result = extract(
+			'<html><body><section id="page-body">Custom main</section></body></html>',
+			'#page-body',
+		);
+
+		expect(result.wordCount).toBe(10);
+		expect(result.main?.id).toBe('page-body');
+	});
+
+	it('falls back to built-in selectors when custom selector is invalid', () => {
+		const result = extract(
+			'<html><body><main>Still works</main></body></html>',
+			'[[[invalid',
+		);
+
+		expect(result.main?.nodeName).toBe('MAIN');
+		expect(result.wordCount).toBe(10);
+	});
+
+	it('extracts headings from main content', () => {
+		const result = extract(`
+			<html><body><main>
+				<h1>Title</h1>
+				<h2>Subtitle</h2>
+				<h3>Section</h3>
+			</main></body></html>
+		`);
+
+		expect(result.headings).toEqual([
+			{ text: 'Title', level: 1 },
+			{ text: 'Subtitle', level: 2 },
+			{ text: 'Section', level: 3 },
+		]);
+	});
+
+	it('extracts images from main content', () => {
+		const result = extract(`
+			<html><body><main>
+				<img src="a.png" alt="Image A">
+				<img src="b.png" alt="Image B">
+				<input type="image" src="c.png" alt="Image C">
+			</main></body></html>
+		`);
+
+		expect(result.images).toHaveLength(3);
+		expect(result.images[0]?.alt).toBe('Image A');
+		expect(result.images[0]?.src).toContain('a.png');
+	});
+
+	it('extracts table metadata from main content', () => {
+		const result = extract(`
+			<html><body><main>
+				<table>
+					<thead><tr><th>A</th><th>B</th></tr></thead>
+					<tfoot><tr><td>Footer</td><td></td></tr></tfoot>
+					<tbody>
+						<tr><td colspan="2">Merged</td></tr>
+						<tr><td>1</td><td>2</td></tr>
+					</tbody>
+				</table>
+			</main></body></html>
+		`);
+
+		expect(result.tables).toEqual([
+			{
+				rows: 4,
+				cols: 2,
+				hasHeader: true,
+				hasFooter: true,
+				hasMergedCell: true,
+			},
+		]);
+	});
+
+	it('extracts buttons from main content', () => {
+		const result = extract(`
+			<html><body><main>
+				<button>Save</button>
+				<a role="button" href="#">Go</a>
+				<span class="btn">Click</span>
+				<div class="share-button">Share</div>
+			</main></body></html>
+		`);
+
+		expect(result.buttons).toHaveLength(4);
+		expect(result.buttons[0]).toMatchObject({
+			nodeName: 'BUTTON',
+			text: 'Save',
+			disabled: false,
+		});
+		expect(result.buttons[1]?.role).toBe('button');
+	});
+
+	it('extracts iframe, video, audio, and canvas from main content', () => {
+		const result = extract(`
+			<html><body><main>
+				<iframe src="/embed" title="Embed" width="640" height="360"></iframe>
+				<video src="/movie.mp4" poster="/poster.jpg" width="320" height="180"></video>
+				<audio src="/sound.mp3"></audio>
+				<canvas width="100" height="50"></canvas>
+			</main></body></html>
+		`);
+
+		expect(result.iframes).toHaveLength(1);
+		expect(result.iframes[0]).toMatchObject({
+			title: 'Embed',
+			width: '640',
+			height: '360',
+		});
+		expect(result.iframes[0]?.src).toContain('/embed');
+
+		expect(result.videos).toHaveLength(1);
+		expect(result.videos[0]?.src).toContain('/movie.mp4');
+		expect(result.videos[0]?.poster).toContain('/poster.jpg');
+		expect(result.videos[0]?.width).toBe(320);
+		expect(result.videos[0]?.height).toBe(180);
+
+		expect(result.audios).toHaveLength(1);
+		expect(result.audios[0]?.src).toContain('/sound.mp3');
+
+		expect(result.canvases).toEqual([{ width: 100, height: 50 }]);
+	});
+
+	it('treats whitespace-only main text as zero word count', () => {
+		const result = extract('<html><body><main>   \n\t  </main></body></html>');
+
+		expect(result.wordCount).toBe(0);
+	});
+
+	it('detects #content fallback selector', () => {
+		const result = extract(
+			'<html><body><div id="content">Fallback content</div></body></html>',
+		);
+
+		expect(result.wordCount).toBe(15);
+	});
+
+	it('detects .main fallback selector', () => {
+		const result = extract(
+			'<html><body><div class="main">Class-based main</div></body></html>',
+		);
+
+		expect(result.wordCount).toBe(15);
+	});
+
+	it('returns first matching element in DOM order when multiple Phase-1 selectors match', () => {
+		const result = extract(`
+			<html><body>
+				<main>Semantic main</main>
+				<div id="content">ID content</div>
+			</body></html>
+		`);
+
+		expect(result.main?.nodeName).toBe('MAIN');
+		expect(result.wordCount).toBe(12);
+	});
+
+	it('uses Phase-2 class*=main when Phase-1 finds nothing', () => {
+		const result = extract(
+			'<html><body><div class="page-main-area">Phase two</div></body></html>',
+		);
+
+		expect(result.main?.classList).toContain('page-main-area');
+		// "Phase two" → "Phasetwo" (8) after whitespace removal
+		expect(result.wordCount).toBe(8);
+	});
+
+	it('uses Phase-2 id*=content case-insensitively when Phase-1 finds nothing', () => {
+		const result = extract(
+			'<html><body><div id="PrimaryContent">Primary</div></body></html>',
+		);
+
+		expect(result.main?.id).toBe('PrimaryContent');
+		expect(result.wordCount).toBe(7);
+	});
+
+	it('prefers Phase-1 <main> over an earlier header-content that would match Phase-2', () => {
+		const result = extract(`
+			<html><body>
+				<div class="header-content">Chrome</div>
+				<main>Real main</main>
+			</body></html>
+		`);
+
+		expect(result.main?.nodeName).toBe('MAIN');
+		expect(result.wordCount).toBe(8);
+	});
+
+	it('may match class*=main on maintenance via Phase-2 (accepted false positive)', () => {
+		const result = extract(
+			'<html><body><div class="maintenance">Down</div></body></html>',
+		);
+
+		expect(result.main?.classList).toContain('maintenance');
+		expect(result.wordCount).toBe(4);
+	});
+
+	it('builds a simple diagnostic selector from tag, id, and classes', () => {
+		const result = extract(
+			'<html><body><div id="page-body" class="foo bar" role="main">X</div></body></html>',
+		);
+
+		expect(result.main?.selector).toBe('div#page-body.foo.bar');
+	});
+});
+
+describe('getMainContents', () => {
+	it('passes only the selector into page.evaluate so Document defaults in the page realm', async () => {
+		const evaluate = vi.fn(
+			(fn: typeof extractMainContentsFromDocument, sel: string | null) =>
+				Promise.resolve(
+					fn(sel, createDocument('<html><body><main>Hi</main></body></html>')),
+				),
+		);
+		const page = { evaluate } as unknown as Page;
+
+		const result = await getMainContents(page, { mainContentSelector: '#x' });
+
+		expect(evaluate).toHaveBeenCalledWith(extractMainContentsFromDocument, '#x');
+		expect(result.wordCount).toBe(2);
+	});
+});

package/src/get-main-contents.ts

@@ -0,0 +1,306 @@
+/**
+ * Main-content quantitative extraction for scraped HTML pages.
+ *
+ * Runs entirely against a `Document` (Puppeteer page realm or jsdom) so the
+ * same logic is unit-tested without a browser. Puppeteer callers use
+ * {@link getMainContents}, which passes this function into `page.evaluate`.
+ *
+ * WHY no `@medv/finder`: selector strings are diagnostic only; a tag+id+class // cspell:disable-line
+ * path is enough and avoids a Node-only dependency inside the page realm.
+ * @module
+ */
+
+import type { MainContentsData } from './types.js';
+import type { Page } from 'puppeteer';
+
+/**
+ * Extract main-content metrics from a `Document`.
+ *
+ * This function must remain free of closure over imports so Puppeteer can
+ * serialize it into `page.evaluate`.
+ *
+ * Argument order matches `page.evaluate(fn, mainContentSelector)`: the selector
+ * is the first argument; `doc` defaults to the page-realm global `document`
+ * when omitted (jsdom tests pass an explicit `Document` as the second argument).
+ * @param mainContentSelector - Optional selector prepended to the default list.
+ * @param doc - The document to inspect (defaults to the global `document`).
+ * @returns Quantitative main-content data (never `null`; empty when no main region).
+ * @example
+ * ```ts
+ * // In page.evaluate / browser:
+ * const data = extractMainContentsFromDocument('#page-body');
+ * // In jsdom tests:
+ * const data = extractMainContentsFromDocument('#page-body', document);
+ * ```
+ */
+export function extractMainContentsFromDocument(
+	mainContentSelector: string | null = null,
+	doc: Document = document,
+): MainContentsData {
+	/**
+	 * @param text
+	 */
+	function removeSpaces(text: string | null): string | null {
+		return text?.trim().replaceAll(/\s+/g, '') || null;
+	}
+
+	/**
+	 * @param value
+	 */
+	function cssEscape(value: string): string {
+		if (typeof CSS !== 'undefined' && typeof CSS.escape === 'function') {
+			return CSS.escape(value);
+		}
+		return value.replaceAll(/([ !"#$%&'()*+,./:;<=>?@[\\\]^`{|}~])/g, '\\$1');
+	}
+
+	/**
+	 * @param el
+	 */
+	function buildSelector(el: Element): string {
+		const tag = el.nodeName.toLowerCase();
+		const id = el.id ? `#${cssEscape(el.id)}` : '';
+		const classes = [...el.classList].map((c) => `.${cssEscape(c)}`).join('');
+		return `${tag}${id}${classes}`;
+	}
+
+	/**
+	 * @param media
+	 */
+	function resolveMediaSrc(media: HTMLMediaElement): string {
+		if (media.currentSrc) {
+			return media.currentSrc;
+		}
+		const attrSrc = media.getAttribute('src');
+		if (attrSrc) {
+			try {
+				return new URL(attrSrc, doc.baseURI).href;
+			} catch {
+				return attrSrc;
+			}
+		}
+		const source = media.querySelector('source[src]');
+		const sourceSrc = source?.getAttribute('src');
+		if (sourceSrc) {
+			try {
+				return new URL(sourceSrc, doc.baseURI).href;
+			} catch {
+				return sourceSrc;
+			}
+		}
+		return '';
+	}
+
+	const title = doc.title?.trim() ?? '';
+	const bodyWordCount = removeSpaces(doc.body?.textContent ?? null)?.length ?? 0;
+
+	const selectors = [
+		'main',
+		'[role="main"]',
+		'#main',
+		'.main',
+		'#content',
+		'.content',
+		'#contents',
+		'.contents',
+		'#main-content',
+		'.main-content',
+		'#main_content',
+		'.main_content',
+		'#mainContent',
+		'.mainContent',
+	];
+	if (mainContentSelector) {
+		selectors.unshift(mainContentSelector);
+	}
+
+	let $main: Element | null = null;
+	try {
+		$main = doc.querySelector(selectors.join(','));
+	} catch {
+		// Invalid custom selector: retry without it so built-in selectors still run.
+		if (mainContentSelector) {
+			selectors.shift();
+			$main = doc.querySelector(selectors.join(','));
+		}
+	}
+
+	if (!$main) {
+		const fallbackSelectors = [
+			'[id*="main" i]',
+			'[class*="main" i]',
+			'[id*="content" i]',
+			'[class*="content" i]',
+		];
+		for (const sel of fallbackSelectors) {
+			const candidate = doc.querySelector(sel);
+			if (candidate && candidate !== doc.body && candidate !== doc.documentElement) {
+				$main = candidate;
+				break;
+			}
+		}
+	}
+
+	if (!$main) {
+		return {
+			title,
+			main: null,
+			wordCount: 0,
+			bodyWordCount,
+			headings: [],
+			images: [],
+			tables: [],
+			buttons: [],
+			iframes: [],
+			videos: [],
+			audios: [],
+			canvases: [],
+		};
+	}
+
+	const headings: MainContentsData['headings'] = [];
+	for (const $heading of $main.querySelectorAll<HTMLHeadingElement>(
+		'h1, h2, h3, h4, h5, h6',
+	)) {
+		headings.push({
+			text: removeSpaces($heading.textContent),
+			level: Number.parseInt($heading.nodeName.replace(/h/i, ''), 10) as
+				1 | 2 | 3 | 4 | 5 | 6,
+		});
+	}
+
+	const images: MainContentsData['images'] = [];
+	for (const $img of $main.querySelectorAll<HTMLImageElement>(
+		'img, input[type="image"]',
+	)) {
+		images.push({
+			src: $img.src,
+			alt: $img.alt,
+		});
+	}
+
+	const tables: MainContentsData['tables'] = [];
+	for (const $table of $main.querySelectorAll<HTMLTableElement>('table')) {
+		tables.push({
+			rows: $table.querySelectorAll('tr').length,
+			cols: $table.querySelector('tr')?.querySelectorAll('th, td').length || 0,
+			hasHeader: !!$table.querySelector('thead'),
+			hasFooter: !!$table.querySelector('tfoot'),
+			hasMergedCell: !!$table.querySelector('[colspan], [rowspan]'),
+		});
+	}
+
+	const win = doc.defaultView;
+	const HTMLButton = win?.HTMLButtonElement;
+	const HTMLInput = win?.HTMLInputElement;
+
+	const buttons: MainContentsData['buttons'] = [];
+	for (const $el of $main.querySelectorAll(
+		'button, [role="button"], [class*="button"], [class*="btn"]',
+	)) {
+		const isButton = HTMLButton ? $el instanceof HTMLButton : $el.nodeName === 'BUTTON';
+		const isInput = HTMLInput ? $el instanceof HTMLInput : $el.nodeName === 'INPUT';
+		const isFormControl = isButton || isInput;
+		let text = removeSpaces($el.textContent);
+		if (!text && isInput) {
+			text = removeSpaces($el.getAttribute('value'));
+		}
+		const disabledProp =
+			isFormControl && 'disabled' in $el
+				? Boolean(($el as HTMLButtonElement).disabled)
+				: false;
+		buttons.push({
+			nodeName: $el.nodeName,
+			role: $el.getAttribute('role'),
+			type: isFormControl ? ($el as HTMLButtonElement | HTMLInputElement).type : null,
+			text,
+			disabled: disabledProp || $el.getAttribute('aria-disabled') === 'true',
+		});
+	}
+
+	const iframes: MainContentsData['iframes'] = [];
+	for (const $el of $main.querySelectorAll<HTMLIFrameElement>('iframe')) {
+		iframes.push({
+			src: $el.src,
+			title: $el.hasAttribute('title') ? $el.getAttribute('title') : null,
+			width: $el.getAttribute('width'),
+			height: $el.getAttribute('height'),
+		});
+	}
+
+	const videos: MainContentsData['videos'] = [];
+	for (const $el of $main.querySelectorAll<HTMLVideoElement>('video')) {
+		const posterAttr = $el.getAttribute('poster');
+		let poster: string | null = null;
+		if (posterAttr) {
+			try {
+				poster = new URL(posterAttr, doc.baseURI).href;
+			} catch {
+				poster = posterAttr;
+			}
+		}
+		videos.push({
+			src: resolveMediaSrc($el),
+			poster,
+			width: $el.width,
+			height: $el.height,
+		});
+	}
+
+	const audios: MainContentsData['audios'] = [];
+	for (const $el of $main.querySelectorAll<HTMLAudioElement>('audio')) {
+		audios.push({
+			src: resolveMediaSrc($el),
+		});
+	}
+
+	const canvases: MainContentsData['canvases'] = [];
+	for (const $el of $main.querySelectorAll<HTMLCanvasElement>('canvas')) {
+		canvases.push({
+			width: $el.width,
+			height: $el.height,
+		});
+	}
+
+	return {
+		title,
+		main: {
+			nodeName: $main.nodeName,
+			id: $main.id || null,
+			classList: [...$main.classList],
+			role: $main.getAttribute('role'),
+			selector: buildSelector($main),
+		},
+		wordCount: removeSpaces($main.textContent)?.length ?? 0,
+		bodyWordCount,
+		headings,
+		images,
+		tables,
+		buttons,
+		iframes,
+		videos,
+		audios,
+		canvases,
+	};
+}
+
+/**
+ * Extract main-content metrics from a Puppeteer page via a single `page.evaluate`.
+ * @param page - Puppeteer page whose DOM has finished loading.
+ * @param options - Optional main-content selector override.
+ * @param options.mainContentSelector
+ * @returns Quantitative main-content data.
+ * @example
+ * ```ts
+ * const mainContents = await getMainContents(page, { mainContentSelector: '#page-body' });
+ * ```
+ */
+export async function getMainContents(
+	page: Page,
+	options?: { mainContentSelector?: string | null },
+): Promise<MainContentsData> {
+	return page.evaluate(
+		extractMainContentsFromDocument,
+		options?.mainContentSelector ?? null,
+	);
+}

package/src/index.ts

@@ -27,6 +27,17 @@ export type {
 	ImageElement,
 	SkippedPageData,
 	NetworkLog,
+	ScrollHeightData,
+	MainContentsData,
+	MainContentsMainTag,
+	MainContentsHeading,
+	MainContentsImage,
+	MainContentsTable,
+	MainContentsButton,
+	MainContentsIframe,
+	MainContentsVideo,
+	MainContentsAudio,
+	MainContentsCanvas,
 	OpenGraphMeta,
 	OgArticleMeta,
 	OgBookMeta,

package/src/is-html-content-type.spec.ts

@@ -0,0 +1,17 @@
+import { describe, expect, it } from 'vitest';
+
+import { isHtmlContentType } from './is-html-content-type.js';
+
+describe('isHtmlContentType', () => {
+	it('accepts text/html in any letter case and with surrounding whitespace', () => {
+		expect(isHtmlContentType('text/html')).toBe(true);
+		expect(isHtmlContentType('Text/HTML')).toBe(true);
+		expect(isHtmlContentType(' text/html ')).toBe(true);
+	});
+
+	it('rejects non-HTML and null', () => {
+		expect(isHtmlContentType('application/pdf')).toBe(false);
+		expect(isHtmlContentType('text/plain')).toBe(false);
+		expect(isHtmlContentType(null)).toBe(false);
+	});
+});

package/src/is-html-content-type.ts

@@ -0,0 +1,20 @@
+/**
+ * Determine whether a Content-Type media type is HTML.
+ *
+ * MIME types are case-insensitive (RFC 2045). Surrounding whitespace after
+ * parameter stripping is also tolerated so `text/html `, `Text/HTML`, and
+ * `text/html; charset=utf-8` (after `;` split) all classify as HTML.
+ * @param contentType - The media type portion of a Content-Type header
+ *   (parameters already stripped), or `null` when unknown.
+ * @returns `true` when the media type is `text/html` in any letter case.
+ * @example
+ * ```ts
+ * isHtmlContentType('text/html'); // true
+ * isHtmlContentType('Text/HTML'); // true
+ * isHtmlContentType('application/pdf'); // false
+ * // Callers should strip parameters first (e.g. split on ';') before calling.
+ * ```
+ */
+export function isHtmlContentType(contentType: string | null): boolean {
+	return contentType !== null && contentType.trim().toLowerCase() === 'text/html';
+}

package/src/measure-scroll-height.spec.ts

@@ -0,0 +1,47 @@
+import type { Page } from 'puppeteer';
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { measureScrollHeight } from './measure-scroll-height.js';
+
+describe('measureScrollHeight', () => {
+	let page: Page;
+	let scrollHeights: number[];
+
+	beforeEach(() => {
+		scrollHeights = [2400, 5200];
+		let call = 0;
+		page = {
+			setViewport: vi.fn(() => Promise.resolve()),
+			evaluate: vi.fn(() => Promise.resolve(scrollHeights[call++] ?? 0)),
+		} as unknown as Page;
+	});
+
+	it('returns desktop and mobile heights without scrolling the page', async () => {
+		const result = await measureScrollHeight(page);
+
+		expect(result).toEqual({ desktop: 2400, mobile: 5200 });
+		expect(page.setViewport).toHaveBeenCalledTimes(2);
+		expect(page.setViewport).toHaveBeenNthCalledWith(1, {
+			width: 1280,
+			height: 800,
+			deviceScaleFactor: 1,
+		});
+		expect(page.setViewport).toHaveBeenNthCalledWith(2, {
+			width: 320,
+			height: 800,
+			deviceScaleFactor: 2,
+		});
+	});
+
+	it('returns null for a preset that throws', async () => {
+		page.setViewport = vi
+			.fn()
+			.mockResolvedValueOnce()
+			.mockRejectedValueOnce(new Error('viewport failed'));
+
+		const result = await measureScrollHeight(page);
+
+		expect(result).toEqual({ desktop: 2400, mobile: null });
+	});
+});