@d-zero/beholder 0.1.28 → 2.0.0

package/dist/sub-process-runner.js

@@ -1,180 +0,0 @@
-import childProcess from 'node:child_process';
-import path from 'node:path';
-import { delay } from '@d-zero/shared/delay';
-import { TypedAwaitEventEmitter } from '@d-zero/shared/typed-await-event-emitter';
-import { isType } from 'typescript-fsa';
-import { scraperLog } from './debug.js';
-import { scraperEvent, subProcessEvent } from './events.js';
-const __filename = new globalThis.URL(import.meta.url).pathname;
-const __dirname = path.dirname(__filename);
-const SUB_PROCESS_PATH = path.resolve(__dirname, 'sub-process');
-export default class SubProcessRunner extends TypedAwaitEventEmitter {
-    #resetTime;
-    #scrapedTimes = 0;
-    #state = 'waiting';
-    #subProcess = null;
-    #undeadPid = new Set();
-    get state() {
-        return this.#state;
-    }
-    constructor(resetTime) {
-        super();
-        this.#resetTime = resetTime;
-    }
-    destory() {
-        const pid = this.#subProcess?.pid;
-        if (this.#subProcess) {
-            scraperLog('Destroys child_process (%d)', pid);
-            this.#subProcess.send(subProcessEvent.destroy());
-            if (pid) {
-                void this.emit('reset', {
-                    pid,
-                });
-                void this.emit('changePhase', {
-                    pid,
-                    name: 'reset',
-                    url: null,
-                    isExternal: false,
-                    message: 'Reseting sub-process',
-                });
-            }
-            this.#destroyed();
-            return;
-        }
-        this.#destroyed();
-    }
-    getUndeadPid() {
-        return [...this.#undeadPid];
-    }
-    kill() {
-        if (!this.#subProcess) {
-            return;
-        }
-        if (!this.#subProcess.killed) {
-            scraperLog('Kills(SIGKILL) child_process (%d) ', this.#subProcess.pid);
-            this.#subProcess.kill('SIGKILL');
-            return;
-        }
-        scraperLog('child_process(%d) is already killed', this.#subProcess.pid);
-    }
-    start(url, options, isSkip, interval) {
-        if (this.#state === 'running') {
-            throw new Error(`Sub Routine (PID: ${this.#subProcess?.pid}) is already running`);
-        }
-        void this.#scrape(url, options, isSkip, interval);
-    }
-    #createSubProcess(url) {
-        const subProcess = childProcess.fork(SUB_PROCESS_PATH, {
-            detached: false,
-        });
-        subProcess.on('message', (action) => {
-            if (isType(action, scraperEvent.changePhase)) {
-                void this.emit('changePhase', action.payload);
-            }
-            if (isType(action, scraperEvent.ignoreAndSkip)) {
-                this.#finished();
-            }
-            if (isType(action, scraperEvent.scrapeEnd)) {
-                this.#finished();
-            }
-            if (isType(action, scraperEvent.destroyed)) {
-                this.#destroyed();
-            }
-            if (isType(action, scraperEvent.ignoreAndSkip) ||
-                isType(action, scraperEvent.resourceResponse) ||
-                isType(action, scraperEvent.scrapeEnd) ||
-                isType(action, scraperEvent.destroyed)) {
-                void this.emit('scrapeEvent', action);
-            }
-            if (isType(action, scraperEvent.error)) {
-                const error = new Error(action.payload.error.message);
-                error.name = action.payload.error.name;
-                error.stack = action.payload.error.stack;
-                const _action = {
-                    type: action.type,
-                    payload: {
-                        pid: action.payload.pid,
-                        url: action.payload.url,
-                        shutdown: action.payload.shutdown,
-                        error,
-                    },
-                };
-                void this.emit('scrapeEvent', _action);
-            }
-        });
-        subProcess.on('disconnect', () => {
-            if (subProcess.killed) {
-                scraperLog('child_process(%d) is disconnected and killed', subProcess.pid);
-                return;
-            }
-            scraperLog('child_process(%d) is disconnected but not killed', subProcess.pid);
-            scraperLog('Retries to kill(SIGTERM) child_process(%d)', subProcess.pid);
-            subProcess.kill('SIGTERM');
-            void this.emit('changePhase', {
-                pid: subProcess.pid,
-                name: 'disconnect',
-                url: null,
-                isExternal: false,
-                message: 'Disconnecting sub-process',
-            });
-        });
-        subProcess.on('error', (e) => {
-            void this.emit('error', {
-                pid: subProcess.pid,
-                url,
-                shutdown: true,
-                error: e instanceof Error ? e : new Error(`${e}`),
-            });
-        });
-        return subProcess;
-    }
-    #destroyed() {
-        if (this.#subProcess && !this.#subProcess.killed && this.#subProcess.pid) {
-            scraperLog('Add child_process(%d) to the undead PID list', this.#subProcess.pid);
-            this.#undeadPid.add(this.#subProcess.pid);
-        }
-        this.#subProcess = null;
-        this.#scrapedTimes = 0;
-        this.#finally();
-    }
-    #finally() {
-        this.#state = 'waiting';
-    }
-    #finished() {
-        this.#scrapedTimes += 1;
-        if (this.#scrapedTimes >= this.#resetTime) {
-            this.destory();
-        }
-        else {
-            this.#finally();
-        }
-    }
-    async #scrape(url, options, isSkip, interval) {
-        if (!this.#subProcess) {
-            this.#subProcess = this.#createSubProcess(url);
-            void this.emit('changePhase', {
-                pid: this.#subProcess.pid,
-                name: 'boot',
-                url,
-                isExternal: options.isExternal,
-                message: 'Booting sub-process',
-            });
-        }
-        this.#state = 'running';
-        interval = Math.max(interval, 0);
-        if (interval) {
-            await delay(interval);
-        }
-        this.#subProcess.send(subProcessEvent.start({
-            url,
-            isExternal: options.isExternal,
-            isGettingImages: options.isGettingImages,
-            excludeKeywords: options.excludeKeywords,
-            executablePath: options.executablePath,
-            disableQueries: options.disableQueries ?? false,
-            isSkip,
-            isTitleOnly: options.isTitleOnly,
-            screenshot: options.screenshot,
-        }));
-    }
-}

package/dist/sub-process.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/sub-process.js

@@ -1,67 +0,0 @@
-// Run on child_process
-import { isType } from 'typescript-fsa';
-import { scraperLog } from './debug.js';
-import { scraperEvent, subProcessEvent } from './events.js';
-import Scraper from './scraper.js';
-const log = scraperLog.extend(`${process.pid}`);
-process.title = 'beholder';
-const scraper = new Scraper();
-scraper.on('resourceResponse', (context) => {
-    if (process.connected) {
-        process.send(scraperEvent.resourceResponse(context));
-    }
-});
-scraper.on('ignoreAndSkip', (context) => {
-    if (process.connected) {
-        process.send(scraperEvent.ignoreAndSkip(context));
-    }
-});
-scraper.on('scrapeEnd', (context) => {
-    if (process.connected) {
-        process.send(scraperEvent.scrapeEnd(context));
-    }
-});
-scraper.on('error', (context) => {
-    if (process.connected) {
-        const _context = {
-            ...context,
-            error: {
-                name: context.error.name,
-                message: context.error.message,
-                stack: context.error.stack,
-            },
-        };
-        process.send(scraperEvent.error(_context));
-    }
-});
-scraper.on('changePhase', (context) => {
-    if (process.connected) {
-        process.send(scraperEvent.changePhase(context));
-    }
-});
-process.on('message', async (action) => {
-    if (isType(action, subProcessEvent.start)) {
-        void scraper.scrapeStart(action.payload.url, {
-            isExternal: action.payload.isExternal,
-            isGettingImages: action.payload.isGettingImages,
-            excludeKeywords: action.payload.excludeKeywords,
-            executablePath: action.payload.executablePath,
-            disableQueries: action.payload.disableQueries,
-            isTitleOnly: action.payload.isTitleOnly,
-            screenshot: action.payload.screenshot,
-        }, action.payload.isSkip);
-    }
-    if (isType(action, subProcessEvent.destroy)) {
-        await scraper.destroy(false);
-    }
-});
-scraper.on('destroyed', (context) => {
-    if (process.connected) {
-        process.send(scraperEvent.destroyed(context));
-        log('disconnects process');
-        process.disconnect();
-    }
-});
-process.on('disconnect', () => {
-    log('Process is disconnected');
-});

package/dist/utils.d.ts

@@ -1,16 +0,0 @@
-import type { CDNType, CompressType } from './types.js';
-/**
- *
- * @param status
- */
-export declare function isError(status: number): boolean;
-/**
- *
- * @param headers
- */
-export declare function detectCompress(headers: Record<string, string | string[] | undefined>): false | CompressType;
-/**
- *
- * @param headers
- */
-export declare function detectCDN(headers: Record<string, string | string[] | undefined>): false | CDNType;

package/dist/utils.js

@@ -1,69 +0,0 @@
-/**
- *
- * @param status
- */
-export function isError(status) {
-    return !(200 <= status && status < 400);
-}
-/**
- *
- * @param headers
- */
-export function detectCompress(headers) {
-    const enc = 'content-encoding' in headers && typeof headers['content-encoding'] === 'string'
-        ? headers['content-encoding']
-        : '';
-    if (/gzip/i.test(enc)) {
-        return 'gzip';
-    }
-    if (/br/i.test(enc)) {
-        return 'br';
-    }
-    if (/compress/i.test(enc)) {
-        return 'compress';
-    }
-    if (/deflate/i.test(enc)) {
-        return 'deflate';
-    }
-    // cspell:disable-next
-    if (/sdch/i.test(enc)) {
-        // cspell:disable-next
-        return 'sdch';
-    }
-    // cspell:disable-next
-    if (/vcdiff/i.test(enc)) {
-        // cspell:disable-next
-        return 'vcdiff';
-    }
-    // cspell:disable-next
-    if (/xdelta/i.test(enc)) {
-        // cspell:disable-next
-        return 'xdelta';
-    }
-    return false;
-}
-/**
- *
- * @param headers
- */
-export function detectCDN(headers) {
-    if ('X-Akamai-Transformed' in headers) {
-        return 'Akamai';
-    }
-    if ('x-amz-cf-pop' in headers) {
-        return 'Amazon CloudFront';
-    }
-    if ('X-IIJ-Cache' in headers) {
-        return 'IIJ';
-    }
-    if (typeof headers.server === 'string') {
-        if (/cloudflare/i.test(headers.server)) {
-            return 'Cloudflare';
-        }
-        if (/amazons3/i.test(headers.server)) {
-            return 'Amazon S3';
-        }
-        return false;
-    }
-    return false;
-}

package/src/events.ts

@@ -1,21 +0,0 @@
-import type { ScrapeEventTypes, SubProcessEventTypes } from './types.js';
-
-import { actionCreatorFactory } from 'typescript-fsa';
-
-const scraperEventCreator = actionCreatorFactory('@@scraper');
-const subProcessEventCreator = actionCreatorFactory('@@sub-process');
-
-export const subProcessEvent = {
-	start: subProcessEventCreator<SubProcessEventTypes['start']>('start'),
-	destroy: subProcessEventCreator<SubProcessEventTypes['destroy']>('destroy'),
-};
-
-export const scraperEvent = {
-	ignoreAndSkip: scraperEventCreator<ScrapeEventTypes['ignoreAndSkip']>('ignoreAndSkip'),
-	resourceResponse:
-		scraperEventCreator<ScrapeEventTypes['resourceResponse']>('resourceResponse'),
-	scrapeEnd: scraperEventCreator<ScrapeEventTypes['scrapeEnd']>('scrapeEnd'),
-	destroyed: scraperEventCreator<ScrapeEventTypes['destroyed']>('destroyed'),
-	error: scraperEventCreator<ScrapeEventTypes['error']>('error'),
-	changePhase: scraperEventCreator<ScrapeEventTypes['changePhase']>('changePhase'),
-};

package/src/fetch-destination.ts

@@ -1,173 +0,0 @@
-import type { PageData, ExURL } from './types.js';
-import type { FollowResponse, RedirectableRequest } from 'follow-redirects';
-import type { ClientRequest, IncomingMessage, RequestOptions } from 'node:http';
-
-import { delay } from '@d-zero/shared/delay';
-import redirects from 'follow-redirects';
-
-import NetTimeoutError from './net-timeout-error.js';
-
-const cacheMap = new Map<string, PageData | Error>();
-
-/**
- *
- * @param url
- * @param isExternal
- * @param method
- */
-export async function fetchDestination(
-	url: ExURL,
-	isExternal: boolean,
-	method = 'HEAD',
-): Promise<PageData> {
-	if (cacheMap.has(url.withoutHash)) {
-		const cache = cacheMap.get(url.withoutHash)!;
-		if (cache instanceof Error) {
-			throw cache;
-		}
-		return cache;
-	}
-
-	const result = await Promise.race([
-		_fetchHead(url, isExternal, method).catch((error) => new Error(error)),
-		(async () => {
-			await delay(10 * 1000);
-			return new NetTimeoutError();
-		})(),
-	]);
-
-	cacheMap.set(url.withoutHash, result);
-	if (result instanceof Error) {
-		throw result;
-	}
-
-	return result;
-}
-
-/**
- *
- * @param url
- * @param isExternal
- * @param method
- */
-async function _fetchHead(url: ExURL, isExternal: boolean, method: string) {
-	return new Promise<PageData>((resolve, reject) => {
-		const hostHeader = url.port ? `${url.hostname}:${url.port}` : url.hostname;
-		const request: RequestOptions = {
-			protocol: url.protocol,
-			hostname: url.hostname,
-			port: url.port || undefined,
-			path: url.pathname,
-			method,
-			headers: {
-				host: hostHeader,
-				Connection: 'keep-alive',
-				Pragma: 'no-cache',
-				'Cache-Control': 'no-cache',
-				'Upgrade-Insecure-Requests': 1,
-				// TODO: 'User-Agent': userAgent,
-				Accept:
-					'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', // cspell:disable-line
-				'Accept-Encoding': 'gzip, deflate',
-				'Accept-Language':
-					'ja,en;q=0.9,zh;q=0.8,en-US;q=0.7,pl;q=0.6,de;q=0.5,zh-CN;q=0.4,zh-TW;q=0.3,th;q=0.2,ko;q=0.1,fr;q=0.1',
-			},
-		};
-
-		if (url.username && url.password) {
-			request.auth = `${url.username}:${url.password}`;
-		}
-
-		let req: RedirectableRequest<ClientRequest, IncomingMessage>;
-		const response = (res: IncomingMessage & FollowResponse) => {
-			res.on('data', () => {});
-			res.on('end', async () => {
-				const redirectPaths = res.redirects.map((r) => r.url);
-				const _contentLength = Number.parseInt(res.headers['content-length'] || '');
-				const contentLength = Number.isFinite(_contentLength) ? _contentLength : null;
-				let rep: PageData = {
-					url,
-					isTarget: !isExternal,
-					isExternal,
-					redirectPaths,
-					status: res.statusCode || 0,
-					statusText: res.statusMessage || '',
-					contentType: res.headers['content-type']?.split(';')[0] || null,
-					contentLength,
-					responseHeaders: res.headers,
-					meta: {
-						title: '',
-					},
-					imageList: [],
-					anchorList: [],
-					html: '',
-					isSkipped: false,
-				};
-
-				if (rep.status === 405) {
-					if (method === 'GET') {
-						reject(`Method Not Allowed: ${url} ${rep.statusText}`);
-						return;
-					}
-					const rr = await fetchDestination(url, isExternal, 'GET').catch(
-						(error) => error,
-					);
-					if (rr) {
-						rep = rr;
-					} else {
-						reject(rr);
-					}
-				}
-
-				if (rep.status === 501) {
-					if (method === 'GET') {
-						reject(`Method Not Implemented: ${url} ${rep.statusText}`);
-						return;
-					}
-					await delay(5 * 1000);
-					const rr = await fetchDestination(url, isExternal, 'GET').catch(
-						(error) => error,
-					);
-					if (rr) {
-						rep = rr;
-					} else {
-						reject(rr);
-					}
-				}
-
-				if (rep.status === 503) {
-					if (method === 'GET') {
-						reject(`Retrying failed: ${url} ${rep.statusText}`);
-						return;
-					}
-					await delay(5 * 1000);
-					const rr = await fetchDestination(url, isExternal, 'GET').catch(
-						(error) => error,
-					);
-					if (rr) {
-						rep = rr;
-					} else {
-						reject(rr);
-					}
-				}
-
-				resolve(rep);
-			});
-		};
-		if (url.protocol === 'https:') {
-			req = redirects.https.request(
-				{
-					...request,
-					rejectUnauthorized: false,
-				},
-				response,
-			);
-		} else {
-			req = redirects.http.request(request, response);
-		}
-		req.on('error', (error) => {
-			reject(error);
-		});
-		req.end();
-	});
-}

package/src/net-timeout-error.ts

@@ -1,3 +0,0 @@
-export default class NetTimeoutError extends Error {
-	override name = 'NetTimeoutError';
-}