@d-zero/beholder 0.1.29 → 2.0.0

package/src/dom-evaluation.ts

@@ -1,25 +1,45 @@
+/**
+ * DOM evaluation functions for extracting structured data from Puppeteer pages.
+ *
+ * These functions are called by {@link ./scraper.ts | Scraper.#fetchData} to extract
+ * anchors, images, and meta information after page navigation completes.
+ * @see {@link ./types.ts} for the data types returned by these functions
+ */
+
 import type { AnchorData, ImageElement, ParseURLOptions } from './types.js';
 import type { ElementHandle, Page } from 'puppeteer';
 
-import { parseUrl } from '@d-zero/shared/parse-url';
-
 import { domDetailsLog, domLog } from './debug.js';
+import { parseUrl } from './parse-url.js';
 
 const pid = `${process.pid}`;
 const log = domLog.extend(pid);
 const dLog = domDetailsLog.extend(pid);
 
 /**
+ * Parameters for {@link getProp}.
+ * @template T - The expected type of the property value.
+ */
+export interface GetPropParams<T> {
+	/** The Puppeteer element handle to read the property from. */
+	readonly $el: ElementHandle<Element>;
+	/** The name of the DOM property to retrieve (e.g., `"href"`, `"textContent"`). */
+	readonly propName: string;
+	/** The default value to return if the property cannot be read or times out. */
+	readonly fallback: T;
+}
+
+/**
+ * Retrieves a DOM property value from a Puppeteer element handle with a timeout.
  *
- * @param $el
- * @param propName
- * @param fallback
+ * Races the actual property retrieval against a 10-second timeout.
+ * If the property cannot be read or the timeout expires, the fallback value is returned.
+ * @template T - The expected type of the property value.
+ * @param params - Parameters containing the element, property name, and fallback.
+ * @returns The property value, or the fallback if retrieval fails.
  */
-export async function getProp<T>(
-	$el: ElementHandle<Element>,
-	propName: string,
-	fallback: T,
-) {
+export async function getProp<T>(params: GetPropParams<T>) {
+	const { $el, propName, fallback } = params;
 	return Promise.race([
 		_getProp($el, propName, fallback),
 		new Promise<T>((res) => setTimeout(() => res(fallback), 10 * 1000)),
@@ -27,10 +47,12 @@ export async function getProp<T>(
 }
 
 /**
- *
- * @param $el
- * @param propName
- * @param fallback
+ * Internal implementation of property retrieval without timeout.
+ * @template T - The expected type of the property value.
+ * @param $el - The Puppeteer element handle.
+ * @param propName - The DOM property name.
+ * @param fallback - The default value on failure.
+ * @returns The property value cast to `T`, or the fallback.
  */
 async function _getProp<T>($el: ElementHandle<Element>, propName: string, fallback: T) {
 	try {
@@ -46,30 +68,46 @@ async function _getProp<T>($el: ElementHandle<Element>, propName: string, fallba
 }
 
 /**
+ * Parameters for {@link getPropBySelector}.
+ * @template T - The expected type of the property value.
+ */
+export interface GetPropBySelectorParams<T> {
+	/** The Puppeteer page to query. */
+	readonly page: Page;
+	/** A CSS selector to find the target element. */
+	readonly selector: string;
+	/** The DOM property name to read from the matched element. */
+	readonly propName: string;
+	/** The default value if no element matches or the property cannot be read. */
+	readonly fallback: T;
+}
+
+/**
+ * Retrieves a DOM property value from the first element matching a CSS selector.
  *
- * @param page
- * @param selector
- * @param propName
- * @param fallback
+ * Combines `page.$()` with {@link getProp} for convenient single-element lookups.
+ * @template T - The expected type of the property value.
+ * @param params - Parameters containing the page, selector, property name, and fallback.
+ * @returns The property value, or the fallback if the element is not found or retrieval fails.
  */
-export async function getPropBySelector<T>(
-	page: Page,
-	selector: string,
-	propName: string,
-	fallback: T,
-) {
+export async function getPropBySelector<T>(params: GetPropBySelectorParams<T>) {
+	const { page, selector, propName, fallback } = params;
 	const $el = await page.$(selector);
 	if (!$el) {
 		return fallback;
 	}
 
-	return getProp($el, propName, fallback);
+	return getProp({ $el, propName, fallback });
 }
 
 /**
+ * Extracts all `<img>` elements from the page and returns their properties.
  *
- * @param page
- * @param viewportWidth
+ * For each image, collects the `src`, `currentSrc`, `alt`, bounding box dimensions,
+ * natural dimensions, lazy-loading status, and the outer HTML source code.
+ * @param page - The Puppeteer page to extract images from.
+ * @param viewportWidth - The current viewport width in pixels, recorded alongside each image entry.
+ * @returns An array of {@link ImageElement} objects describing each image on the page.
  */
 export async function getImageList(
 	page: Page,
@@ -94,13 +132,29 @@ export async function getImageList(
 		const boundingBox = await $image.boundingBox();
 		const width = boundingBox?.width || 0;
 		const height = boundingBox?.height || 0;
-		const src = await getProp($image, 'src', '');
-		const currentSrc = await getProp($image, 'currentSrc', '');
-		const alt = await getProp($image, 'alt', '');
-		const naturalWidth = await getProp($image, 'naturalWidth', 0);
-		const naturalHeight = await getProp($image, 'naturalHeight', 0);
-		const loading = await getProp($image, 'loading', '');
-		const sourceCode = await getProp($image, 'outerHTML', '');
+		const src = await getProp({ $el: $image, propName: 'src', fallback: '' });
+		const currentSrc = await getProp({
+			$el: $image,
+			propName: 'currentSrc',
+			fallback: '',
+		});
+		const alt = await getProp({ $el: $image, propName: 'alt', fallback: '' });
+		const naturalWidth = await getProp({
+			$el: $image,
+			propName: 'naturalWidth',
+			fallback: 0,
+		});
+		const naturalHeight = await getProp({
+			$el: $image,
+			propName: 'naturalHeight',
+			fallback: 0,
+		});
+		const loading = await getProp({ $el: $image, propName: 'loading', fallback: '' });
+		const sourceCode = await getProp({
+			$el: $image,
+			propName: 'outerHTML',
+			fallback: '',
+		});
 		const isLazy = loading.toLowerCase().trim() === 'lazy';
 		imageList.push({
 			src,
@@ -125,9 +179,14 @@ export async function getImageList(
 }
 
 /**
+ * Extracts all anchor (`<a>` and `<area>`) elements with `href` attributes from the page.
  *
- * @param page
- * @param options
+ * For each anchor, resolves the `href` to an `ExURL` via `parseUrl`, retrieves
+ * the accessible name (from the accessibility tree, falling back to `textContent`),
+ * and filters out non-HTTP links.
+ * @param page - The Puppeteer page to extract anchors from.
+ * @param options - Optional URL parsing options (e.g., `disableQueries`).
+ * @returns An array of {@link AnchorData} objects for all HTTP(S) links found on the page.
  */
 export async function getAnchorList(page: Page, options?: ParseURLOptions) {
 	log('Getting anchors');
@@ -136,14 +195,18 @@ export async function getAnchorList(page: Page, options?: ParseURLOptions) {
 	const anchorList: AnchorData[] = [];
 
 	for (const $anchor of $anchors) {
-		const $href = await getProp($anchor, 'href', '');
+		const $href = await getProp({ $el: $anchor, propName: 'href', fallback: '' });
 		const hrefVal = $href.toString();
 		const href = parseUrl(hrefVal, options);
 		if (!href || !href.isHTTP) {
 			continue;
 		}
 		const axNode = await page.accessibility.snapshot({ root: $anchor });
-		const textContent = await getProp($anchor, 'textContent', '');
+		const textContent = await getProp({
+			$el: $anchor,
+			propName: 'textContent',
+			fallback: '',
+		});
 		const accessibleName = axNode ? axNode.name || '' : textContent.trim();
 		const link: AnchorData = {
 			href,
@@ -161,46 +224,113 @@ export async function getAnchorList(page: Page, options?: ParseURLOptions) {
 }
 
 /**
+ * Extracts comprehensive meta information from the page's `<head>`.
  *
- * @param page
+ * Collects the following metadata:
+ * - `title` - The document title.
+ * - `lang` - The `lang` attribute of the `<html>` element.
+ * - `description` - The `<meta name="description">` content.
+ * - `keywords` - The `<meta name="keywords">` content.
+ * - `noindex` / `nofollow` / `noarchive` - Parsed from the `<meta name="robots">` directives.
+ * - `canonical` - The `<link rel="canonical">` content.
+ * - `alternate` - The `<link rel="alternate">` content.
+ * - Open Graph tags: `og:type`, `og:title`, `og:site_name`, `og:description`, `og:url`, `og:image`.
+ * - `twitter:card` - The Twitter Card type.
+ * @param page - The Puppeteer page to extract meta information from.
+ * @returns An object containing all extracted meta properties.
  */
 export async function getMeta(page: Page) {
 	log('Getting Meta');
 
-	const robotsVal = await getPropBySelector(page, 'meta[name="robots"]', 'content', '');
+	const robotsVal = await getPropBySelector({
+		page,
+		selector: 'meta[name="robots"]',
+		propName: 'content',
+		fallback: '',
+	});
 	const robots = new Set(robotsVal.split(',').map((robot) => robot.trim().toLowerCase()));
 	const meta = {
-		title: await getPropBySelector(page, 'title', 'textContent', ''),
-		lang: await getPropBySelector(page, 'html', 'lang', ''),
-		description: await getPropBySelector(page, 'meta[name="description"]', 'content', ''),
-		keywords: await getPropBySelector(page, 'meta[name="keywords"]', 'content', ''),
+		title: await getPropBySelector({
+			page,
+			selector: 'title',
+			propName: 'textContent',
+			fallback: '',
+		}),
+		lang: await getPropBySelector({
+			page,
+			selector: 'html',
+			propName: 'lang',
+			fallback: '',
+		}),
+		description: await getPropBySelector({
+			page,
+			selector: 'meta[name="description"]',
+			propName: 'content',
+			fallback: '',
+		}),
+		keywords: await getPropBySelector({
+			page,
+			selector: 'meta[name="keywords"]',
+			propName: 'content',
+			fallback: '',
+		}),
 		noindex: robots.has('noindex'),
 		nofollow: robots.has('nofollow'),
 		noarchive: robots.has('noarchive'),
-		canonical: await getPropBySelector(page, 'link[rel="canonical"]', 'content', ''),
-		alternate: await getPropBySelector(page, 'link[rel="alternate"]', 'content', ''),
-		'og:type': await getPropBySelector(page, 'meta[property="og:type"]', 'content', ''),
-		'og:title': await getPropBySelector(page, 'meta[property="og:title"]', 'content', ''),
-		'og:site_name': await getPropBySelector(
+		canonical: await getPropBySelector({
+			page,
+			selector: 'link[rel="canonical"]',
+			propName: 'href',
+			fallback: '',
+		}),
+		alternate: await getPropBySelector({
+			page,
+			selector: 'link[rel="alternate"]',
+			propName: 'href',
+			fallback: '',
+		}),
+		'og:type': await getPropBySelector({
+			page,
+			selector: 'meta[property="og:type"]',
+			propName: 'content',
+			fallback: '',
+		}),
+		'og:title': await getPropBySelector({
+			page,
+			selector: 'meta[property="og:title"]',
+			propName: 'content',
+			fallback: '',
+		}),
+		'og:site_name': await getPropBySelector({
+			page,
+			selector: 'meta[property="og:site_name"]',
+			propName: 'content',
+			fallback: '',
+		}),
+		'og:description': await getPropBySelector({
+			page,
+			selector: 'meta[property="og:description"]',
+			propName: 'content',
+			fallback: '',
+		}),
+		'og:url': await getPropBySelector({
 			page,
-			'meta[property="og:site_name"]',
-			'content',
-			'',
-		),
-		'og:description': await getPropBySelector(
+			selector: 'meta[property="og:url"]',
+			propName: 'content',
+			fallback: '',
+		}),
+		'og:image': await getPropBySelector({
 			page,
-			'meta[property="og:description"]',
-			'content',
-			'',
-		),
-		'og:url': await getPropBySelector(page, 'meta[property="og:url"]', 'content', ''),
-		'og:image': await getPropBySelector(page, 'meta[property="og:image"]', 'content', ''),
-		'twitter:card': await getPropBySelector(
+			selector: 'meta[property="og:image"]',
+			propName: 'content',
+			fallback: '',
+		}),
+		'twitter:card': await getPropBySelector({
 			page,
-			'meta[name="twitter:card"]',
-			'content',
-			'',
-		),
+			selector: 'meta[name="twitter:card"]',
+			propName: 'content',
+			fallback: '',
+		}),
 	};
 
 	log('Got meta');

package/src/index.ts

@@ -1,4 +1,28 @@
-export { default as SubProcessRunner } from './sub-process-runner.js';
+/**
+ * @module @d-zero/beholder
+ *
+ * The beholder package provides page-level scraping capabilities for web crawlers.
+ * It handles browser page navigation, DOM data extraction (anchors, images, meta tags),
+ * network resource monitoring, and keyword-based page exclusion.
+ *
+ * Results are returned as values from `scrapeStart()`, not emitted as events.
+ * Only streaming events (changePhase, resourceResponse) are emitted for progress monitoring.
+ *
+ * The main entry point is the `Scraper` class (default export).
+ */
 export { default as default } from './scraper.js';
-export * from './types.js';
-export * from './events.js';
+export { isError } from './is-error.js';
+export { detectCompress } from '@d-zero/shared/detect-compress';
+export type { CompressType } from '@d-zero/shared/detect-compress';
+export { detectCDN } from '@d-zero/shared/detect-cdn';
+export type { CDNType } from '@d-zero/shared/detect-cdn';
+export type { ScrapeResult, ResourceEntry, PageData } from './types.js';
+export type { ScraperOptions, ChangePhaseEvent, ScraperEventTypes } from './types.js';
+export type {
+	Resource,
+	AnchorData,
+	Meta,
+	ImageElement,
+	SkippedPageData,
+	NetworkLog,
+} from './types.js';

package/src/is-error.spec.ts

@@ -0,0 +1,33 @@
+import { describe, it, expect } from 'vitest';
+
+import { isError } from './is-error.js';
+
+describe('isError', () => {
+	it('returns true for status below 200', () => {
+		expect(isError(199)).toBe(true);
+	});
+
+	it('returns false for status 200 (lower boundary)', () => {
+		expect(isError(200)).toBe(false);
+	});
+
+	it('returns false for status 399 (upper boundary)', () => {
+		expect(isError(399)).toBe(false);
+	});
+
+	it('returns true for status 400', () => {
+		expect(isError(400)).toBe(true);
+	});
+
+	it('returns true for status 0', () => {
+		expect(isError(0)).toBe(true);
+	});
+
+	it('returns true for negative status', () => {
+		expect(isError(-1)).toBe(true);
+	});
+
+	it('returns true for status 500', () => {
+		expect(isError(500)).toBe(true);
+	});
+});

package/src/is-error.ts

@@ -0,0 +1,10 @@
+/**
+ * Determines whether an HTTP status code represents an error.
+ * Status codes in the range 200-399 (inclusive) are considered successful;
+ * all others are considered errors.
+ * @param status - HTTP status code to evaluate
+ * @returns `true` if the status code indicates an error (< 200 or >= 400)
+ */
+export function isError(status: number) {
+	return !(200 <= status && status < 400);
+}

package/src/keyword-check.spec.ts

@@ -1,8 +1,49 @@
-import { test, expect } from 'vitest';
+import { describe, it, expect } from 'vitest';
 
 import { keywordCheck } from './keyword-check.js';
 
-test('keyword checking', () => {
-	expect(keywordCheck('abc', ['abc'])).toBe('abc');
-	expect(keywordCheck('ABC', ['abc', '/abc/i'])).toBe('/abc/i');
+describe('keywordCheck', () => {
+	it('returns the matched keyword when found', () => {
+		expect(keywordCheck('<html><body>error message</body></html>', ['error'])).toBe(
+			'error',
+		);
+	});
+
+	it('returns false when no keyword matches', () => {
+		expect(keywordCheck('<html><body>hello world</body></html>', ['error'])).toBe(false);
+	});
+
+	it('returns the first matching keyword when multiple match', () => {
+		expect(
+			keywordCheck('<html><body>error warning</body></html>', ['warning', 'error']),
+		).toBe('warning');
+	});
+
+	it('returns false for empty keyword array', () => {
+		expect(keywordCheck('<html><body>some content</body></html>', [])).toBe(false);
+	});
+
+	it('returns false for empty HTML', () => {
+		expect(keywordCheck('', ['error'])).toBe(false);
+	});
+
+	it('supports regex pattern with /pattern/ syntax', () => {
+		expect(keywordCheck('<html><body>code 404 found</body></html>', ['/\\d{3}/'])).toBe(
+			'/\\d{3}/',
+		);
+	});
+
+	it('is case-sensitive by default for plain keywords', () => {
+		expect(keywordCheck('<html><body>Error</body></html>', ['error'])).toBe(false);
+	});
+
+	it('supports case-insensitive flag /pattern/i', () => {
+		expect(keywordCheck('<html><body>Error</body></html>', ['/error/i'])).toBe(
+			'/error/i',
+		);
+	});
+
+	it('returns false when regex pattern does not match', () => {
+		expect(keywordCheck('<html><body>no numbers</body></html>', ['/\\d+/'])).toBe(false);
+	});
 });

package/src/keyword-check.ts

@@ -1,9 +1,11 @@
 import { strToRegex } from '@d-zero/shared/str-to-regex';
 
 /**
- *
- * @param html
- * @param excludeKeywords
+ * Checks whether the given HTML content contains any of the specified exclude keywords.
+ * Each keyword is converted to a regular expression via `strToRegex` before testing.
+ * @param html - The raw HTML string to search within.
+ * @param excludeKeywords - An array of keyword strings or regex patterns to match against the HTML.
+ * @returns The first matched keyword string if a match is found, or `false` if none match.
  */
 export function keywordCheck(html: string, excludeKeywords: string[]) {
 	for (const keyword of excludeKeywords) {

package/src/parse-url.spec.ts

@@ -0,0 +1,35 @@
+import { describe, it, expect } from 'vitest';
+
+import { parseUrl } from './parse-url.js';
+
+describe('parseUrl', () => {
+	it('parses a string URL into ExURL', () => {
+		const result = parseUrl('https://example.com/path');
+		expect(result).not.toBeNull();
+		expect(result!.hostname).toBe('example.com');
+	});
+
+	it('returns ExURL object as-is when passed an ExURL', () => {
+		const exUrl = parseUrl('https://example.com')!;
+		const result = parseUrl(exUrl);
+		expect(result).toBe(exUrl);
+	});
+
+	it('returns null for fragment-only string', () => {
+		const result = parseUrl('#fragment');
+		expect(result).toBeNull();
+	});
+
+	it('returns non-null for tel: URL (has protocol)', () => {
+		const result = parseUrl('tel:000-0000-0000');
+		// tel: URL has protocol set, so parseUrl does not filter it out
+		expect(result).not.toBeNull();
+		expect(result!.protocol).toBe('tel:');
+	});
+
+	it('parses http URL', () => {
+		const result = parseUrl('http://example.com');
+		expect(result).not.toBeNull();
+		expect(result!.protocol).toBe('http:');
+	});
+});

package/src/parse-url.ts

@@ -0,0 +1,26 @@
+import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
+
+import { parseUrl as sharedParseUrl } from '@d-zero/shared/parse-url';
+
+/**
+ * Parses a URL string into an ExURL object, filtering out non-HTTP URLs
+ * that lack a hostname and protocol. If the input is already an ExURL object,
+ * it is returned as-is without re-parsing.
+ *
+ * WHY null return: Bare fragment-only strings (e.g. `"#section"`) and
+ * protocol-relative paths without a host are not meaningful URLs for crawling.
+ * @param url - A URL string or an already-parsed ExURL object
+ * @param options - URL parsing options (e.g. `disableQueries` to strip query strings)
+ * @returns The parsed ExURL, or `null` if the URL is not navigable
+ * @see `@d-zero/shared/parse-url` for the underlying parser
+ */
+export function parseUrl(url: string | ExURL, options?: ParseURLOptions): ExURL | null {
+	if (typeof url !== 'string') {
+		return url;
+	}
+	const result = sharedParseUrl(url, options);
+	if (!result.isHTTP && !result.hostname && !result.protocol) {
+		return null;
+	}
+	return result;
+}