@d-zero/beholder 0.1.0

package/src/sub-process-runner.ts

@@ -0,0 +1,220 @@
+import type { ScraperOptions } from './scraper.js';
+import type { SubProcessRunnerEventTypes, ExURL } from './types.js';
+import type { AnyAction } from 'typescript-fsa';
+
+import childProcess from 'node:child_process';
+import path from 'node:path';
+
+import { delay } from '@d-zero/shared/delay';
+import { TypedAwaitEventEmitter } from '@d-zero/shared/typed-await-event-emitter';
+import { isType } from 'typescript-fsa';
+
+import { scraperLog } from './debug.js';
+import { scraperEvent, subProcessEvent } from './events.js';
+
+const __filename = new global.URL(import.meta.url).pathname;
+const __dirname = path.dirname(__filename);
+const SUB_PROCESS_PATH = path.resolve(__dirname, 'sub-process');
+
+export default class SubProcessRunner extends TypedAwaitEventEmitter<SubProcessRunnerEventTypes> {
+	readonly #resetTime: number;
+	#scrapedTimes = 0;
+	#state: 'waiting' | 'running' = 'waiting';
+	#subProcess: childProcess.ChildProcess | null = null;
+	#undeadPid = new Set<number>();
+
+	get state() {
+		return this.#state;
+	}
+
+	constructor(resetTime: number) {
+		super();
+		this.#resetTime = resetTime;
+	}
+
+	destory() {
+		const pid = this.#subProcess?.pid;
+		if (this.#subProcess) {
+			scraperLog('Destroys child_process (%d)', pid);
+			this.#subProcess.send(subProcessEvent.destroy());
+
+			if (pid) {
+				void this.emit('reset', {
+					pid,
+				});
+				void this.emit('changePhase', {
+					pid,
+					name: 'reset',
+					url: null,
+					isExternal: false,
+					message: 'Reseting sub-process',
+				});
+			}
+
+			this.#destroyed();
+			return;
+		}
+		this.#destroyed();
+	}
+
+	getUndeadPid() {
+		return [...this.#undeadPid];
+	}
+
+	kill() {
+		if (!this.#subProcess) {
+			return;
+		}
+		if (!this.#subProcess.killed) {
+			scraperLog('Kills(SIGKILL) child_process (%d) ', this.#subProcess.pid);
+			this.#subProcess.kill('SIGKILL');
+			return;
+		}
+		scraperLog('child_process(%d) is already killed', this.#subProcess.pid);
+	}
+
+	start(url: ExURL, options: ScraperOptions, isSkip: boolean, interval: number) {
+		if (this.#state === 'running') {
+			throw new Error(`Sub Routine (PID: ${this.#subProcess?.pid}) is already running`);
+		}
+		void this.#scrape(url, options, isSkip, interval);
+	}
+
+	#createSubProcess(url: ExURL) {
+		const subProcess = childProcess.fork(SUB_PROCESS_PATH, {
+			detached: false,
+		});
+
+		subProcess.on('message', (action: AnyAction) => {
+			if (isType(action, scraperEvent.changePhase)) {
+				void this.emit('changePhase', action.payload);
+			}
+
+			if (isType(action, scraperEvent.ignoreAndSkip)) {
+				this.#finished();
+			}
+
+			if (isType(action, scraperEvent.scrapeEnd)) {
+				this.#finished();
+			}
+
+			if (isType(action, scraperEvent.destroyed)) {
+				this.#destroyed();
+			}
+
+			if (
+				isType(action, scraperEvent.ignoreAndSkip) ||
+				isType(action, scraperEvent.resourceResponse) ||
+				isType(action, scraperEvent.scrapeEnd) ||
+				isType(action, scraperEvent.destroyed)
+			) {
+				void this.emit('scrapeEvent', action);
+			}
+
+			if (isType(action, scraperEvent.error)) {
+				const error = new Error(action.payload.error.message);
+				error.name = action.payload.error.name;
+				error.stack = action.payload.error.stack;
+				const _action = {
+					type: action.type,
+					payload: {
+						pid: action.payload.pid,
+						url: action.payload.url,
+						shutdown: action.payload.shutdown,
+						error,
+					},
+				};
+				void this.emit('scrapeEvent', _action);
+			}
+		});
+
+		subProcess.on('disconnect', () => {
+			if (subProcess.killed) {
+				scraperLog('child_process(%d) is disconnected and killed', subProcess.pid);
+				return;
+			}
+
+			scraperLog('child_process(%d) is disconnected but not killed', subProcess.pid);
+			scraperLog('Retries to kill(SIGTERM) child_process(%d)', subProcess.pid);
+			subProcess.kill('SIGTERM');
+
+			void this.emit('changePhase', {
+				pid: subProcess.pid,
+				name: 'disconnect',
+				url: null,
+				isExternal: false,
+				message: 'Disconnecting sub-process',
+			});
+		});
+
+		subProcess.on('error', (e) => {
+			void this.emit('error', {
+				pid: subProcess.pid,
+				url,
+				shutdown: true,
+				error: e instanceof Error ? e : new Error(`${e}`),
+			});
+		});
+
+		return subProcess;
+	}
+
+	#destroyed() {
+		if (this.#subProcess && !this.#subProcess.killed && this.#subProcess.pid) {
+			scraperLog('Add child_process(%d) to the undead PID list', this.#subProcess.pid);
+			this.#undeadPid.add(this.#subProcess.pid);
+		}
+		this.#subProcess = null;
+		this.#scrapedTimes = 0;
+		this.#finally();
+	}
+
+	#finally() {
+		this.#state = 'waiting';
+	}
+
+	#finished() {
+		this.#scrapedTimes += 1;
+
+		if (this.#scrapedTimes >= this.#resetTime) {
+			this.destory();
+		} else {
+			this.#finally();
+		}
+	}
+
+	async #scrape(url: ExURL, options: ScraperOptions, isSkip: boolean, interval: number) {
+		if (!this.#subProcess) {
+			this.#subProcess = this.#createSubProcess(url);
+
+			void this.emit('changePhase', {
+				pid: this.#subProcess.pid,
+				name: 'boot',
+				url,
+				isExternal: options.isExternal,
+				message: 'Booting sub-process',
+			});
+		}
+
+		this.#state = 'running';
+
+		interval = Math.max(interval, 0);
+		if (interval) {
+			await delay(interval);
+		}
+
+		this.#subProcess.send(
+			subProcessEvent.start({
+				url,
+				isExternal: options.isExternal,
+				isGettingImages: options.isGettingImages,
+				excludeKeywords: options.excludeKeywords,
+				executablePath: options.executablePath,
+				disableQueries: options.disableQueries ?? false,
+				isSkip,
+				isTitleOnly: options.isTitleOnly,
+				screenshot: options.screenshot,
+			}),
+		);
+	}
+}

package/src/sub-process.ts

@@ -0,0 +1,86 @@
+// Run on child_process
+
+import type { AnyAction } from 'typescript-fsa';
+
+import { isType } from 'typescript-fsa';
+
+import { scraperLog } from './debug.js';
+import { scraperEvent, subProcessEvent } from './events.js';
+import Scraper from './scraper.js';
+
+const log = scraperLog.extend(`${process.pid}`);
+
+process.title = 'beholder';
+const scraper = new Scraper();
+
+scraper.on('resourceResponse', (context) => {
+	if (process.connected) {
+		process.send!(scraperEvent.resourceResponse(context));
+	}
+});
+
+scraper.on('ignoreAndSkip', (context) => {
+	if (process.connected) {
+		process.send!(scraperEvent.ignoreAndSkip(context));
+	}
+});
+
+scraper.on('scrapeEnd', (context) => {
+	if (process.connected) {
+		process.send!(scraperEvent.scrapeEnd(context));
+	}
+});
+
+scraper.on('error', (context) => {
+	if (process.connected) {
+		const _context = {
+			...context,
+			error: {
+				name: context.error.name,
+				message: context.error.message,
+				stack: context.error.stack,
+			},
+		};
+		process.send!(scraperEvent.error(_context));
+	}
+});
+
+scraper.on('changePhase', (context) => {
+	if (process.connected) {
+		process.send!(scraperEvent.changePhase(context));
+	}
+});
+
+process.on('message', async (action: AnyAction) => {
+	if (isType(action, subProcessEvent.start)) {
+		void scraper.scrapeStart(
+			action.payload.url,
+			{
+				isExternal: action.payload.isExternal,
+				isGettingImages: action.payload.isGettingImages,
+				excludeKeywords: action.payload.excludeKeywords,
+				executablePath: action.payload.executablePath,
+				disableQueries: action.payload.disableQueries,
+				isTitleOnly: action.payload.isTitleOnly,
+				screenshot: action.payload.screenshot,
+			},
+			action.payload.isSkip,
+		);
+	}
+
+	if (isType(action, subProcessEvent.destroy)) {
+		await scraper.destroy(false);
+	}
+});
+
+scraper.on('destroyed', (context) => {
+	if (process.connected) {
+		process.send!(scraperEvent.destroyed(context));
+		log('disconnects process');
+		process.disconnect();
+	}
+});
+
+process.on('disconnect', () => {
+	log('Process is disconnected');
+});

package/src/types.ts

@@ -0,0 +1,341 @@
+import type { Action } from 'typescript-fsa';
+
+export type ScrapeEvent = {
+	pid: number | undefined;
+	url: ExURL;
+};
+
+export type ScrapeErrorEvent = ScrapeEvent & {
+	shutdown: boolean;
+	error: {
+		name: string;
+		message: string;
+		stack?: string;
+	};
+};
+
+export type ScrapeEventTypes = {
+	ignoreAndSkip: ScrapeEvent & {
+		reason: {
+			matchedText: string;
+			excludeKeywords: string[];
+		};
+	};
+	resourceResponse: ScrapeEvent & {
+		log: NetworkLog;
+		resource: Omit<Resource, 'uid'>;
+	};
+	scrapeEnd: ScrapeEvent & {
+		timestamp: number;
+		result: PageData;
+	};
+	destroyed: Omit<ScrapeEvent, 'url'>;
+	error: ScrapeErrorEvent;
+	changePhase: ChangePhaseEvent;
+};
+
+export type ChangePhaseEvent = {
+	pid: number;
+	name:
+		| 'scrapeStart'
+		| 'launchBrowser'
+		| 'touchHead'
+		| 'touchHeadTimeout'
+		| 'newPage'
+		| 'openPage'
+		| 'loadDOMContent'
+		| 'waitNetworkIdleZero'
+		| 'getHTML'
+		| 'setViewport'
+		| 'scrollToBottom'
+		| 'getImages'
+		| 'getAnchors'
+		| 'getMeta'
+		| 'ignoreAndSkip'
+		| 'scrapeEnd'
+		| 'beforeDestroy'
+		| 'destroyed';
+	url: ExURL | null;
+	isExternal: boolean;
+	message: string;
+};
+
+export type AnyScrapeEvent = ScrapeEventTypes[keyof ScrapeEventTypes];
+
+export type SubProcessEventTypes = {
+	start: {
+		url: ExURL;
+		isExternal: boolean;
+		isGettingImages: boolean;
+		excludeKeywords: string[];
+		executablePath: string | null;
+		isSkip: boolean;
+		isTitleOnly: boolean;
+		screenshot: string | null;
+	} & Required<ParseURLOptions>;
+	destroy: void;
+};
+
+export type SubProcessEvent = {
+	pid: number | undefined;
+};
+
+export type SubProcessChangeEvent =
+	| ChangePhaseEvent
+	| {
+			pid: number | undefined;
+			name: 'reset' | 'boot' | 'disconnect';
+			url: ExURL | null;
+			isExternal: boolean;
+			message: string;
+	  };
+
+export type SubProcessRunnerEventTypes = {
+	reset: SubProcessEvent;
+	scrapeEvent: Action<AnyScrapeEvent>;
+	changePhase: SubProcessChangeEvent;
+	error: ScrapeErrorEvent;
+};
+
+export type ExURL = {
+	/**
+	 * Full URL (optimized)
+	 */
+	href: string;
+
+	/**
+	 * Full URL that before parse
+	 */
+	_originUrlString: string;
+
+	/**
+	 * Full URL without hash
+	 */
+	withoutHash: string;
+
+	/**
+	 * Full URL without hash and Authentication
+	 */
+	withoutHashAndAuth: string;
+
+	/**
+	 * Protocol or URI scheme (includes ":")
+	 * - case-insensitive
+	 */
+	protocol: string;
+
+	/**
+	 * Whether protocol is HTTP or HTTPS
+	 */
+	isHTTP: boolean;
+
+	/**
+	 * Whether protocol is HTTPS
+	 */
+	isSecure: boolean;
+
+	/**
+	 * User name of authentication
+	 */
+	username: string | null;
+
+	/**
+	 * Password of authentication
+	 */
+	password: string | null;
+
+	/**
+	 * Host name
+	 *
+	 * - case-insensitive
+	 * - encode non-ASCII characters
+	 * - without port number
+	 */
+	hostname: string;
+
+	/**
+	 * Port number
+	 */
+	port: string | null;
+
+	/**
+	 * Path part
+	 *
+	 * It is only `/` if pathname is empty
+	 *
+	 * - case-sensitive
+	 */
+	pathname: string | null;
+
+	/**
+	 * Array of path
+	 */
+	paths: string[];
+
+	/**
+	 * Depth of paths
+	 */
+	depth: number;
+
+	/**
+	 * Directory name of paths
+	 *
+	 * It is null if it is `/` only
+	 */
+	dirname: string | null;
+
+	/**
+	 * Base name of paths (File name without file extension)
+	 */
+	basename: string | null;
+
+	/**
+	 * Whether index page (It's true if basename is null)
+	 */
+	isIndex: boolean;
+
+	/**
+	 * File extension name (inclues ".")
+	 */
+	extname: string | null;
+
+	/**
+	 * Search query (without `?`)
+	 *
+	 * - case-sensitive
+	 */
+	query: string | null;
+
+	/**
+	 * Hash (includes `#`)
+	 *
+	 * - case-sensitive
+	 */
+	hash: string | null;
+};
+
+export type ParseURLOptions = {
+	disableQueries?: boolean;
+};
+
+export type PageData = {
+	url: ExURL;
+	redirectPaths: string[];
+	isTarget: boolean;
+	isExternal: boolean;
+	status: number;
+	statusText: string;
+	contentType: string | null;
+	contentLength: number | null;
+	responseHeaders: Record<string, string | string[] | undefined> | null;
+	meta: Meta;
+	anchorList: AnchorData[];
+	imageList: ImageElement[];
+	html: string;
+	isSkipped: false;
+};
+
+export type Meta = {
+	lang?: string;
+	title: string;
+	description?: string;
+	keywords?: string;
+	noindex?: boolean;
+	nofollow?: boolean;
+	noarchive?: boolean;
+	canonical?: string;
+	alternate?: string;
+	'og:type'?: string;
+	'og:title'?: string;
+	'og:site_name'?: string;
+	'og:description'?: string;
+	'og:url'?: string;
+	'og:image'?: string;
+	'twitter:card'?: string;
+};
+
+export type AnchorData = {
+	/**
+	 * Extracts the value of the `href` attribute from anchor element (`<a>` `<area>`)
+	 */
+	href: ExURL;
+
+	/**
+	 * The accessible name of the anchor element
+	 */
+	textContent: string;
+};
+
+export type ImageElement = {
+	src: string;
+	currentSrc: string;
+	alt: string;
+	width: number;
+	height: number;
+	naturalWidth: number;
+	naturalHeight: number;
+	isLazy: boolean;
+	viewportWidth: number;
+	sourceCode: string;
+};
+
+export type NetworkLog = {
+	url: ExURL;
+	status: number | null;
+	contentLength: number;
+	contentType: string;
+	isError: boolean;
+	request: {
+		ts: number;
+		headers: Record<string, string>;
+		method: string;
+	};
+	response?: {
+		ts: number;
+		status: number;
+		statusText: string;
+		fromCache: boolean;
+		headers: Record<string, string>;
+	};
+};
+
+export type Resource = {
+	url: ExURL;
+	isExternal: boolean;
+	isError: boolean;
+	status: number | null;
+	statusText: string | null;
+	contentType: string | null;
+	contentLength: number | null;
+	compress: false | CompressType;
+	cdn: false | CDNType;
+	headers: Record<string, string | string[] | undefined> | null;
+};
+
+export type CompressType =
+	| 'gzip'
+	| 'compress'
+	| 'deflate'
+	| 'br'
+	| 'sdch' // cspell:disable-line
+	| 'vcdiff' // cspell:disable-line
+	| 'xdelta'; // cspell:disable-line
+
+export type CDNType = 'Amazon S3' | 'Amazon CloudFront' | 'IIJ' | 'Cloudflare' | 'Akamai';
+
+export type HTTPMethod = 'HEAD' | 'GET' | 'POST' | 'PATCH' | 'PUT' | 'DELETE' | 'OPTIONS';
+
+export type SkippedPageData = {
+	isSkipped: true;
+	url: ExURL;
+	matched:
+		| {
+				type: 'keyword';
+				text: string;
+				excludeKeywords: string[];
+		  }
+		| {
+				type: 'path';
+				excludes: string[];
+		  };
+};