@d-zero/beholder 0.1.0

package/package.json

@@ -0,0 +1,34 @@
+{
+	"name": "@d-zero/beholder",
+	"version": "0.1.0",
+	"description": "The tool for scraping and recording web pages",
+	"author": "D-ZERO",
+	"license": "MIT",
+	"private": false,
+	"publishConfig": {
+		"access": "public"
+	},
+	"type": "module",
+	"exports": {
+		".": {
+			"import": "./lib/index.js",
+			"types": "./lib/index.d.ts"
+		}
+	},
+	"scripts": {
+		"build": "tsc",
+		"clean": "tsc --build --clean"
+	},
+	"dependencies": {
+		"@d-zero/shared": "0.4.0",
+		"debug": "4.3.4",
+		"follow-redirects": "1.15.6",
+		"puppeteer": "22.9.0",
+		"typescript-fsa": "3.0.0"
+	},
+	"devDependencies": {
+		"@types/debug": "4.1.12",
+		"@types/follow-redirects": "1.14.4"
+	},
+	"gitHead": "acfefd8b20364c2aec1998517fc0b190fdc3b404"
+}

package/src/debug.ts

@@ -0,0 +1,7 @@
+import debug from 'debug';
+
+export const log = debug('Beholder');
+export const scraperLog = log.extend('Scraper');
+export const resourceLog = scraperLog.extend('Resource');
+export const domLog = scraperLog.extend('DOM');
+export const domDetailsLog = domLog.extend('Details');

package/src/dom-evaluation.ts

@@ -0,0 +1,175 @@
+import type { AnchorData, ImageElement, ParseURLOptions } from './types.js';
+import type { ElementHandle, Page } from 'puppeteer';
+
+import { domDetailsLog, domLog } from './debug.js';
+import { parseUrl } from './utils.js';
+
+const pid = `${process.pid}`;
+const log = domLog.extend(pid);
+const dLog = domDetailsLog.extend(pid);
+
+export async function getProp<T>(
+	$el: ElementHandle<Element>,
+	propName: string,
+	fallback: T,
+) {
+	return Promise.race([
+		_getProp($el, propName, fallback),
+		new Promise<T>((res) => setTimeout(() => res(fallback), 10 * 1000)),
+	]);
+}
+
+async function _getProp<T>($el: ElementHandle<Element>, propName: string, fallback: T) {
+	try {
+		const prop = await $el.getProperty(propName);
+		if (!prop) {
+			return fallback;
+		}
+		const value = (await prop.jsonValue()) as T;
+		return value;
+	} catch {
+		return fallback;
+	}
+}
+
+export async function getPropBySelector<T>(
+	page: Page,
+	selector: string,
+	propName: string,
+	fallback: T,
+) {
+	const $el = await page.$(selector);
+	if (!$el) {
+		return fallback;
+	}
+
+	return getProp($el, propName, fallback);
+}
+
+export async function getImageList(
+	page: Page,
+	viewportWidth: number,
+): Promise<ImageElement[]> {
+	log('Getting images (Viewport: %dpx)', viewportWidth);
+
+	const $images = await page.$$('img');
+	const imageList: {
+		src: string;
+		currentSrc: string;
+		alt: string;
+		width: number;
+		height: number;
+		naturalWidth: number;
+		naturalHeight: number;
+		isLazy: boolean;
+		viewportWidth: number;
+		sourceCode: string;
+	}[] = [];
+	for (const $image of $images) {
+		const boundingBox = await $image.boundingBox();
+		const width = boundingBox?.width || 0;
+		const height = boundingBox?.height || 0;
+		const src = await getProp($image, 'src', '');
+		const currentSrc = await getProp($image, 'currentSrc', '');
+		const alt = await getProp($image, 'alt', '');
+		const naturalWidth = await getProp($image, 'naturalWidth', 0);
+		const naturalHeight = await getProp($image, 'naturalHeight', 0);
+		const loading = await getProp($image, 'loading', '');
+		const sourceCode = await getProp($image, 'outerHTML', '');
+		const isLazy = loading.toLowerCase().trim() === 'lazy';
+		imageList.push({
+			src,
+			currentSrc,
+			alt,
+			width,
+			height,
+			naturalWidth,
+			naturalHeight,
+			isLazy,
+			viewportWidth,
+			sourceCode,
+		});
+	}
+
+	log('Got %d images (Viewport: %dpx)', imageList.length, viewportWidth);
+	dLog(
+		'Images are: %O',
+		imageList.map((i) => i.src),
+	);
+	return imageList;
+}
+
+export async function getAnchorList(page: Page, options?: ParseURLOptions) {
+	log('Getting anchors');
+
+	const $anchors = await page.$$('a[href], area[href]');
+	const anchorList: AnchorData[] = [];
+
+	for (const $anchor of $anchors) {
+		const $href = await getProp($anchor, 'href', '');
+		const hrefVal = $href.toString();
+		const href = parseUrl(hrefVal, options);
+		if (!href || !href.isHTTP) {
+			continue;
+		}
+		const axNode = await page.accessibility.snapshot({ root: $anchor });
+		const textContent = await getProp($anchor, 'textContent', '');
+		const accessibleName = axNode ? axNode.name || '' : textContent.trim();
+		const link: AnchorData = {
+			href,
+			textContent: accessibleName,
+		};
+		anchorList.push(link);
+	}
+
+	log('Got %d anchors', anchorList.length);
+	dLog(
+		'Anchors are: %O',
+		anchorList.map((a) => a.href.href),
+	);
+	return anchorList;
+}
+
+export async function getMeta(page: Page) {
+	log('Getting Meta');
+
+	const robotsVal = await getPropBySelector(page, 'meta[name="robots"]', 'content', '');
+	const robots = new Set(robotsVal.split(',').map((robot) => robot.trim().toLowerCase()));
+	const meta = {
+		title: await getPropBySelector(page, 'title', 'textContent', ''),
+		lang: await getPropBySelector(page, 'html', 'lang', ''),
+		description: await getPropBySelector(page, 'meta[name="description"]', 'content', ''),
+		keywords: await getPropBySelector(page, 'meta[name="keywords"]', 'content', ''),
+		noindex: robots.has('noindex'),
+		nofollow: robots.has('nofollow'),
+		noarchive: robots.has('noarchive'),
+		canonical: await getPropBySelector(page, 'link[rel="canonical"]', 'content', ''),
+		alternate: await getPropBySelector(page, 'link[rel="alternate"]', 'content', ''),
+		'og:type': await getPropBySelector(page, 'meta[property="og:type"]', 'content', ''),
+		'og:title': await getPropBySelector(page, 'meta[property="og:title"]', 'content', ''),
+		'og:site_name': await getPropBySelector(
+			page,
+			'meta[property="og:site_name"]',
+			'content',
+			'',
+		),
+		'og:description': await getPropBySelector(
+			page,
+			'meta[property="og:description"]',
+			'content',
+			'',
+		),
+		'og:url': await getPropBySelector(page, 'meta[property="og:url"]', 'content', ''),
+		'og:image': await getPropBySelector(page, 'meta[property="og:image"]', 'content', ''),
+		'twitter:card': await getPropBySelector(
+			page,
+			'meta[name="twitter:card"]',
+			'content',
+			'',
+		),
+	};
+
+	log('Got meta');
+	dLog('Meta data are: %O', meta);
+	return meta;
+}

package/src/events.ts

@@ -0,0 +1,21 @@
+import type { ScrapeEventTypes, SubProcessEventTypes } from './types.js';
+
+import { actionCreatorFactory } from 'typescript-fsa';
+
+const scraperEventCreator = actionCreatorFactory('@@scraper');
+const subProcessEventCreator = actionCreatorFactory('@@sub-process');
+
+export const subProcessEvent = {
+	start: subProcessEventCreator<SubProcessEventTypes['start']>('start'),
+	destroy: subProcessEventCreator<SubProcessEventTypes['destroy']>('destroy'),
+};
+
+export const scraperEvent = {
+	ignoreAndSkip: scraperEventCreator<ScrapeEventTypes['ignoreAndSkip']>('ignoreAndSkip'),
+	resourceResponse:
+		scraperEventCreator<ScrapeEventTypes['resourceResponse']>('resourceResponse'),
+	scrapeEnd: scraperEventCreator<ScrapeEventTypes['scrapeEnd']>('scrapeEnd'),
+	destroyed: scraperEventCreator<ScrapeEventTypes['destroyed']>('destroyed'),
+	error: scraperEventCreator<ScrapeEventTypes['error']>('error'),
+	changePhase: scraperEventCreator<ScrapeEventTypes['changePhase']>('changePhase'),
+};

package/src/fetch-destination.ts

@@ -0,0 +1,160 @@
+import type { PageData, ExURL } from './types.js';
+import type { FollowResponse, RedirectableRequest } from 'follow-redirects';
+import type { ClientRequest, IncomingMessage, RequestOptions } from 'node:http';
+
+import { delay } from '@d-zero/shared/delay';
+import redirects from 'follow-redirects';
+
+import NetTimeoutError from './net-timeout-error.js';
+
+const cacheMap = new Map<string, PageData | Error>();
+
+export async function fetchDestination(
+	url: ExURL,
+	isExternal: boolean,
+	method = 'HEAD',
+): Promise<PageData> {
+	if (cacheMap.has(url.withoutHash)) {
+		const cache = cacheMap.get(url.withoutHash)!;
+		if (cache instanceof Error) {
+			throw cache;
+		}
+		return cache;
+	}
+
+	const result = await Promise.race([
+		_fetchHead(url, isExternal, method).catch((error) => new Error(error)),
+		(async () => {
+			await delay(10 * 1000);
+			return new NetTimeoutError();
+		})(),
+	]);
+
+	cacheMap.set(url.withoutHash, result);
+	if (result instanceof Error) {
+		throw result;
+	}
+
+	return result;
+}
+
+async function _fetchHead(url: ExURL, isExternal: boolean, method: string) {
+	return new Promise<PageData>((resolve, reject) => {
+		const request: RequestOptions = {
+			protocol: url.protocol,
+			host: url.hostname,
+			path: url.pathname,
+			method,
+			headers: {
+				host: url.hostname,
+				Connection: 'keep-alive',
+				Pragma: 'no-cache',
+				'Cache-Control': 'no-cache',
+				'Upgrade-Insecure-Requests': 1,
+				// TODO: 'User-Agent': userAgent,
+				Accept:
+					'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', // cspell:disable-line
+				'Accept-Encoding': 'gzip, deflate',
+				'Accept-Language':
+					'ja,en;q=0.9,zh;q=0.8,en-US;q=0.7,pl;q=0.6,de;q=0.5,zh-CN;q=0.4,zh-TW;q=0.3,th;q=0.2,ko;q=0.1,fr;q=0.1',
+			},
+		};
+
+		if (url.username && url.password) {
+			request.auth = `${url.username}:${url.password}`;
+		}
+
+		let req: RedirectableRequest<ClientRequest, IncomingMessage>;
+		const response = (res: IncomingMessage & FollowResponse) => {
+			res.on('data', () => {});
+			res.on('end', async () => {
+				const redirectPaths = res.redirects.map((r) => r.url);
+				const _contentLength = Number.parseInt(res.headers['content-length'] || '');
+				const contentLength = Number.isFinite(_contentLength) ? _contentLength : null;
+				let rep: PageData = {
+					url,
+					isTarget: !isExternal,
+					isExternal,
+					redirectPaths,
+					status: res.statusCode || 0,
+					statusText: res.statusMessage || '',
+					contentType: res.headers['content-type']?.split(';')[0] || null,
+					contentLength,
+					responseHeaders: res.headers,
+					meta: {
+						title: '',
+					},
+					imageList: [],
+					anchorList: [],
+					html: '',
+					isSkipped: false,
+				};
+
+				if (rep.status === 405) {
+					if (method === 'GET') {
+						reject(`Method Not Allowed: ${url} ${rep.statusText}`);
+						return;
+					}
+					const rr = await fetchDestination(url, isExternal, 'GET').catch(
+						(error) => error,
+					);
+					if (rr) {
+						rep = rr;
+					} else {
+						reject(rr);
+					}
+				}
+
+				if (rep.status === 501) {
+					if (method === 'GET') {
+						reject(`Method Not Implemented: ${url} ${rep.statusText}`);
+						return;
+					}
+					await delay(5 * 1000);
+					const rr = await fetchDestination(url, isExternal, 'GET').catch(
+						(error) => error,
+					);
+					if (rr) {
+						rep = rr;
+					} else {
+						reject(rr);
+					}
+				}
+
+				if (rep.status === 503) {
+					if (method === 'GET') {
+						reject(`Retrying failed: ${url} ${rep.statusText}`);
+						return;
+					}
+					await delay(5 * 1000);
+					const rr = await fetchDestination(url, isExternal, 'GET').catch(
+						(error) => error,
+					);
+					if (rr) {
+						rep = rr;
+					} else {
+						reject(rr);
+					}
+				}
+
+				resolve(rep);
+			});
+		};
+		if (url.protocol === 'https:') {
+			req = redirects.https.request(
+				{
+					...request,
+					rejectUnauthorized: false,
+				},
+				response,
+			);
+		} else {
+			req = redirects.http.request(request, response);
+		}
+		req.on('error', (error) => {
+			reject(error);
+		});
+		req.write('head');
+		req.end();
+	});
+}

package/src/index.ts

@@ -0,0 +1,4 @@
+export { default as SubProcessRunner } from './sub-process-runner.js';
+export { default as default } from './scraper.js';
+export * from './types.js';
+export * from './events.js';

package/src/keyword-check.spec.ts

@@ -0,0 +1,8 @@
+import { test, expect } from 'vitest';
+
+import { keywordCheck } from './keyword-check.js';
+
+test('keyword checking', () => {
+	expect(keywordCheck('abc', ['abc'])).toBe('abc');
+	expect(keywordCheck('ABC', ['abc', '/abc/i'])).toBe('/abc/i');
+});

package/src/keyword-check.ts

@@ -0,0 +1,12 @@
+import { strToRegex } from '@d-zero/shared/str-to-regex';
+
+export function keywordCheck(html: string, excludeKeywords: string[]) {
+	for (const keyword of excludeKeywords) {
+		const pattern = strToRegex(keyword);
+		if (pattern.test(html)) {
+			return keyword;
+		}
+	}
+
+	return false;
+}

package/src/net-timeout-error.ts

@@ -0,0 +1,3 @@
+export default class NetTimeoutError extends Error {
+	override name = 'NetTimeoutError';
+}