@d-zero/beholder 0.1.29 → 2.0.1

package/src/fetch-destination.ts

@@ -1,173 +0,0 @@
-import type { PageData, ExURL } from './types.js';
-import type { FollowResponse, RedirectableRequest } from 'follow-redirects';
-import type { ClientRequest, IncomingMessage, RequestOptions } from 'node:http';
-
-import { delay } from '@d-zero/shared/delay';
-import redirects from 'follow-redirects';
-
-import NetTimeoutError from './net-timeout-error.js';
-
-const cacheMap = new Map<string, PageData | Error>();
-
-/**
- *
- * @param url
- * @param isExternal
- * @param method
- */
-export async function fetchDestination(
-	url: ExURL,
-	isExternal: boolean,
-	method = 'HEAD',
-): Promise<PageData> {
-	if (cacheMap.has(url.withoutHash)) {
-		const cache = cacheMap.get(url.withoutHash)!;
-		if (cache instanceof Error) {
-			throw cache;
-		}
-		return cache;
-	}
-
-	const result = await Promise.race([
-		_fetchHead(url, isExternal, method).catch((error) => new Error(error)),
-		(async () => {
-			await delay(10 * 1000);
-			return new NetTimeoutError();
-		})(),
-	]);
-
-	cacheMap.set(url.withoutHash, result);
-	if (result instanceof Error) {
-		throw result;
-	}
-
-	return result;
-}
-
-/**
- *
- * @param url
- * @param isExternal
- * @param method
- */
-async function _fetchHead(url: ExURL, isExternal: boolean, method: string) {
-	return new Promise<PageData>((resolve, reject) => {
-		const hostHeader = url.port ? `${url.hostname}:${url.port}` : url.hostname;
-		const request: RequestOptions = {
-			protocol: url.protocol,
-			hostname: url.hostname,
-			port: url.port || undefined,
-			path: url.pathname,
-			method,
-			headers: {
-				host: hostHeader,
-				Connection: 'keep-alive',
-				Pragma: 'no-cache',
-				'Cache-Control': 'no-cache',
-				'Upgrade-Insecure-Requests': 1,
-				// TODO: 'User-Agent': userAgent,
-				Accept:
-					'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', // cspell:disable-line
-				'Accept-Encoding': 'gzip, deflate',
-				'Accept-Language':
-					'ja,en;q=0.9,zh;q=0.8,en-US;q=0.7,pl;q=0.6,de;q=0.5,zh-CN;q=0.4,zh-TW;q=0.3,th;q=0.2,ko;q=0.1,fr;q=0.1',
-			},
-		};
-
-		if (url.username && url.password) {
-			request.auth = `${url.username}:${url.password}`;
-		}
-
-		let req: RedirectableRequest<ClientRequest, IncomingMessage>;
-		const response = (res: IncomingMessage & FollowResponse) => {
-			res.on('data', () => {});
-			res.on('end', async () => {
-				const redirectPaths = res.redirects.map((r) => r.url);
-				const _contentLength = Number.parseInt(res.headers['content-length'] || '');
-				const contentLength = Number.isFinite(_contentLength) ? _contentLength : null;
-				let rep: PageData = {
-					url,
-					isTarget: !isExternal,
-					isExternal,
-					redirectPaths,
-					status: res.statusCode || 0,
-					statusText: res.statusMessage || '',
-					contentType: res.headers['content-type']?.split(';')[0] || null,
-					contentLength,
-					responseHeaders: res.headers,
-					meta: {
-						title: '',
-					},
-					imageList: [],
-					anchorList: [],
-					html: '',
-					isSkipped: false,
-				};
-
-				if (rep.status === 405) {
-					if (method === 'GET') {
-						reject(`Method Not Allowed: ${url} ${rep.statusText}`);
-						return;
-					}
-					const rr = await fetchDestination(url, isExternal, 'GET').catch(
-						(error) => error,
-					);
-					if (rr) {
-						rep = rr;
-					} else {
-						reject(rr);
-					}
-				}
-
-				if (rep.status === 501) {
-					if (method === 'GET') {
-						reject(`Method Not Implemented: ${url} ${rep.statusText}`);
-						return;
-					}
-					await delay(5 * 1000);
-					const rr = await fetchDestination(url, isExternal, 'GET').catch(
-						(error) => error,
-					);
-					if (rr) {
-						rep = rr;
-					} else {
-						reject(rr);
-					}
-				}
-
-				if (rep.status === 503) {
-					if (method === 'GET') {
-						reject(`Retrying failed: ${url} ${rep.statusText}`);
-						return;
-					}
-					await delay(5 * 1000);
-					const rr = await fetchDestination(url, isExternal, 'GET').catch(
-						(error) => error,
-					);
-					if (rr) {
-						rep = rr;
-					} else {
-						reject(rr);
-					}
-				}
-
-				resolve(rep);
-			});
-		};
-		if (url.protocol === 'https:') {
-			req = redirects.https.request(
-				{
-					...request,
-					rejectUnauthorized: false,
-				},
-				response,
-			);
-		} else {
-			req = redirects.http.request(request, response);
-		}
-		req.on('error', (error) => {
-			reject(error);
-		});
-		req.end();
-	});
-}

package/src/net-timeout-error.ts

@@ -1,3 +0,0 @@
-export default class NetTimeoutError extends Error {
-	override name = 'NetTimeoutError';
-}

package/src/sub-process-runner.ts

@@ -1,220 +0,0 @@
-import type { ScraperOptions } from './scraper.js';
-import type { SubProcessRunnerEventTypes, ExURL } from './types.js';
-import type { AnyAction } from 'typescript-fsa';
-
-import childProcess from 'node:child_process';
-import path from 'node:path';
-
-import { delay } from '@d-zero/shared/delay';
-import { TypedAwaitEventEmitter } from '@d-zero/shared/typed-await-event-emitter';
-import { isType } from 'typescript-fsa';
-
-import { scraperLog } from './debug.js';
-import { scraperEvent, subProcessEvent } from './events.js';
-
-const __filename = new globalThis.URL(import.meta.url).pathname;
-const __dirname = path.dirname(__filename);
-const SUB_PROCESS_PATH = path.resolve(__dirname, 'sub-process');
-
-export default class SubProcessRunner extends TypedAwaitEventEmitter<SubProcessRunnerEventTypes> {
-	readonly #resetTime: number;
-	#scrapedTimes = 0;
-	#state: 'waiting' | 'running' = 'waiting';
-	#subProcess: childProcess.ChildProcess | null = null;
-	#undeadPid = new Set<number>();
-
-	get state() {
-		return this.#state;
-	}
-
-	constructor(resetTime: number) {
-		super();
-		this.#resetTime = resetTime;
-	}
-
-	destory() {
-		const pid = this.#subProcess?.pid;
-		if (this.#subProcess) {
-			scraperLog('Destroys child_process (%d)', pid);
-			this.#subProcess.send(subProcessEvent.destroy());
-
-			if (pid) {
-				void this.emit('reset', {
-					pid,
-				});
-				void this.emit('changePhase', {
-					pid,
-					name: 'reset',
-					url: null,
-					isExternal: false,
-					message: 'Reseting sub-process',
-				});
-			}
-
-			this.#destroyed();
-			return;
-		}
-		this.#destroyed();
-	}
-
-	getUndeadPid() {
-		return [...this.#undeadPid];
-	}
-
-	kill() {
-		if (!this.#subProcess) {
-			return;
-		}
-		if (!this.#subProcess.killed) {
-			scraperLog('Kills(SIGKILL) child_process (%d) ', this.#subProcess.pid);
-			this.#subProcess.kill('SIGKILL');
-			return;
-		}
-		scraperLog('child_process(%d) is already killed', this.#subProcess.pid);
-	}
-
-	start(url: ExURL, options: ScraperOptions, isSkip: boolean, interval: number) {
-		if (this.#state === 'running') {
-			throw new Error(`Sub Routine (PID: ${this.#subProcess?.pid}) is already running`);
-		}
-		void this.#scrape(url, options, isSkip, interval);
-	}
-
-	#createSubProcess(url: ExURL) {
-		const subProcess = childProcess.fork(SUB_PROCESS_PATH, {
-			detached: false,
-		});
-
-		subProcess.on('message', (action: AnyAction) => {
-			if (isType(action, scraperEvent.changePhase)) {
-				void this.emit('changePhase', action.payload);
-			}
-
-			if (isType(action, scraperEvent.ignoreAndSkip)) {
-				this.#finished();
-			}
-
-			if (isType(action, scraperEvent.scrapeEnd)) {
-				this.#finished();
-			}
-
-			if (isType(action, scraperEvent.destroyed)) {
-				this.#destroyed();
-			}
-
-			if (
-				isType(action, scraperEvent.ignoreAndSkip) ||
-				isType(action, scraperEvent.resourceResponse) ||
-				isType(action, scraperEvent.scrapeEnd) ||
-				isType(action, scraperEvent.destroyed)
-			) {
-				void this.emit('scrapeEvent', action);
-			}
-
-			if (isType(action, scraperEvent.error)) {
-				const error = new Error(action.payload.error.message);
-				error.name = action.payload.error.name;
-				error.stack = action.payload.error.stack;
-				const _action = {
-					type: action.type,
-					payload: {
-						pid: action.payload.pid,
-						url: action.payload.url,
-						shutdown: action.payload.shutdown,
-						error,
-					},
-				};
-				void this.emit('scrapeEvent', _action);
-			}
-		});
-
-		subProcess.on('disconnect', () => {
-			if (subProcess.killed) {
-				scraperLog('child_process(%d) is disconnected and killed', subProcess.pid);
-				return;
-			}
-
-			scraperLog('child_process(%d) is disconnected but not killed', subProcess.pid);
-			scraperLog('Retries to kill(SIGTERM) child_process(%d)', subProcess.pid);
-			subProcess.kill('SIGTERM');
-
-			void this.emit('changePhase', {
-				pid: subProcess.pid,
-				name: 'disconnect',
-				url: null,
-				isExternal: false,
-				message: 'Disconnecting sub-process',
-			});
-		});
-
-		subProcess.on('error', (e) => {
-			void this.emit('error', {
-				pid: subProcess.pid,
-				url,
-				shutdown: true,
-				error: e instanceof Error ? e : new Error(`${e}`),
-			});
-		});
-
-		return subProcess;
-	}
-
-	#destroyed() {
-		if (this.#subProcess && !this.#subProcess.killed && this.#subProcess.pid) {
-			scraperLog('Add child_process(%d) to the undead PID list', this.#subProcess.pid);
-			this.#undeadPid.add(this.#subProcess.pid);
-		}
-		this.#subProcess = null;
-		this.#scrapedTimes = 0;
-		this.#finally();
-	}
-
-	#finally() {
-		this.#state = 'waiting';
-	}
-
-	#finished() {
-		this.#scrapedTimes += 1;
-
-		if (this.#scrapedTimes >= this.#resetTime) {
-			this.destory();
-		} else {
-			this.#finally();
-		}
-	}
-
-	async #scrape(url: ExURL, options: ScraperOptions, isSkip: boolean, interval: number) {
-		if (!this.#subProcess) {
-			this.#subProcess = this.#createSubProcess(url);
-
-			void this.emit('changePhase', {
-				pid: this.#subProcess.pid,
-				name: 'boot',
-				url,
-				isExternal: options.isExternal,
-				message: 'Booting sub-process',
-			});
-		}
-
-		this.#state = 'running';
-
-		interval = Math.max(interval, 0);
-		if (interval) {
-			await delay(interval);
-		}
-
-		this.#subProcess.send(
-			subProcessEvent.start({
-				url,
-				isExternal: options.isExternal,
-				isGettingImages: options.isGettingImages,
-				excludeKeywords: options.excludeKeywords,
-				executablePath: options.executablePath,
-				disableQueries: options.disableQueries ?? false,
-				isSkip,
-				isTitleOnly: options.isTitleOnly,
-				screenshot: options.screenshot,
-			}),
-		);
-	}
-}

package/src/sub-process.ts

@@ -1,86 +0,0 @@
-// Run on child_process
-
-import type { AnyAction } from 'typescript-fsa';
-
-import { isType } from 'typescript-fsa';
-
-import { scraperLog } from './debug.js';
-import { scraperEvent, subProcessEvent } from './events.js';
-import Scraper from './scraper.js';
-
-const log = scraperLog.extend(`${process.pid}`);
-
-process.title = 'beholder';
-const scraper = new Scraper();
-
-scraper.on('resourceResponse', (context) => {
-	if (process.connected) {
-		process.send!(scraperEvent.resourceResponse(context));
-	}
-});
-
-scraper.on('ignoreAndSkip', (context) => {
-	if (process.connected) {
-		process.send!(scraperEvent.ignoreAndSkip(context));
-	}
-});
-
-scraper.on('scrapeEnd', (context) => {
-	if (process.connected) {
-		process.send!(scraperEvent.scrapeEnd(context));
-	}
-});
-
-scraper.on('error', (context) => {
-	if (process.connected) {
-		const _context = {
-			...context,
-			error: {
-				name: context.error.name,
-				message: context.error.message,
-				stack: context.error.stack,
-			},
-		};
-		process.send!(scraperEvent.error(_context));
-	}
-});
-
-scraper.on('changePhase', (context) => {
-	if (process.connected) {
-		process.send!(scraperEvent.changePhase(context));
-	}
-});
-
-process.on('message', async (action: AnyAction) => {
-	if (isType(action, subProcessEvent.start)) {
-		void scraper.scrapeStart(
-			action.payload.url,
-			{
-				isExternal: action.payload.isExternal,
-				isGettingImages: action.payload.isGettingImages,
-				excludeKeywords: action.payload.excludeKeywords,
-				executablePath: action.payload.executablePath,
-				disableQueries: action.payload.disableQueries,
-				isTitleOnly: action.payload.isTitleOnly,
-				screenshot: action.payload.screenshot,
-			},
-			action.payload.isSkip,
-		);
-	}
-
-	if (isType(action, subProcessEvent.destroy)) {
-		await scraper.destroy(false);
-	}
-});
-
-scraper.on('destroyed', (context) => {
-	if (process.connected) {
-		process.send!(scraperEvent.destroyed(context));
-		log('disconnects process');
-		process.disconnect();
-	}
-});
-
-process.on('disconnect', () => {
-	log('Process is disconnected');
-});

package/src/utils.ts

@@ -1,89 +0,0 @@
-import type { CDNType, CompressType } from './types.js';
-
-/**
- *
- * @param status
- */
-export function isError(status: number) {
-	return !(200 <= status && status < 400);
-}
-
-/**
- *
- * @param headers
- */
-export function detectCompress(
-	headers: Record<string, string | string[] | undefined>,
-): false | CompressType {
-	const enc =
-		'content-encoding' in headers && typeof headers['content-encoding'] === 'string'
-			? headers['content-encoding']
-			: '';
-	if (/gzip/i.test(enc)) {
-		return 'gzip';
-	}
-
-	if (/br/i.test(enc)) {
-		return 'br';
-	}
-
-	if (/compress/i.test(enc)) {
-		return 'compress';
-	}
-
-	if (/deflate/i.test(enc)) {
-		return 'deflate';
-	}
-
-	// cspell:disable-next
-	if (/sdch/i.test(enc)) {
-		// cspell:disable-next
-		return 'sdch';
-	}
-
-	// cspell:disable-next
-	if (/vcdiff/i.test(enc)) {
-		// cspell:disable-next
-		return 'vcdiff';
-	}
-
-	// cspell:disable-next
-	if (/xdelta/i.test(enc)) {
-		// cspell:disable-next
-		return 'xdelta';
-	}
-
-	return false;
-}
-
-/**
- *
- * @param headers
- */
-export function detectCDN(
-	headers: Record<string, string | string[] | undefined>,
-): false | CDNType {
-	if ('X-Akamai-Transformed' in headers) {
-		return 'Akamai';
-	}
-
-	if ('x-amz-cf-pop' in headers) {
-		return 'Amazon CloudFront';
-	}
-
-	if ('X-IIJ-Cache' in headers) {
-		return 'IIJ';
-	}
-
-	if (typeof headers.server === 'string') {
-		if (/cloudflare/i.test(headers.server)) {
-			return 'Cloudflare';
-		}
-		if (/amazons3/i.test(headers.server)) {
-			return 'Amazon S3';
-		}
-		return false;
-	}
-
-	return false;
-}