@d-zero/beholder 0.1.0

package/dist/sub-process-runner.d.ts

@@ -0,0 +1,12 @@
+import type { ScraperOptions } from './scraper.js';
+import type { SubProcessRunnerEventTypes, ExURL } from './types.js';
+import { TypedAwaitEventEmitter } from '@d-zero/shared/typed-await-event-emitter';
+export default class SubProcessRunner extends TypedAwaitEventEmitter<SubProcessRunnerEventTypes> {
+    #private;
+    get state(): "waiting" | "running";
+    constructor(resetTime: number);
+    destory(): void;
+    getUndeadPid(): number[];
+    kill(): void;
+    start(url: ExURL, options: ScraperOptions, isSkip: boolean, interval: number): void;
+}

package/dist/sub-process-runner.js

@@ -0,0 +1,180 @@
+import childProcess from 'node:child_process';
+import path from 'node:path';
+import { delay } from '@d-zero/shared/delay';
+import { TypedAwaitEventEmitter } from '@d-zero/shared/typed-await-event-emitter';
+import { isType } from 'typescript-fsa';
+import { scraperLog } from './debug.js';
+import { scraperEvent, subProcessEvent } from './events.js';
+const __filename = new global.URL(import.meta.url).pathname;
+const __dirname = path.dirname(__filename);
+const SUB_PROCESS_PATH = path.resolve(__dirname, 'sub-process');
+export default class SubProcessRunner extends TypedAwaitEventEmitter {
+    #resetTime;
+    #scrapedTimes = 0;
+    #state = 'waiting';
+    #subProcess = null;
+    #undeadPid = new Set();
+    get state() {
+        return this.#state;
+    }
+    constructor(resetTime) {
+        super();
+        this.#resetTime = resetTime;
+    }
+    destory() {
+        const pid = this.#subProcess?.pid;
+        if (this.#subProcess) {
+            scraperLog('Destroys child_process (%d)', pid);
+            this.#subProcess.send(subProcessEvent.destroy());
+            if (pid) {
+                void this.emit('reset', {
+                    pid,
+                });
+                void this.emit('changePhase', {
+                    pid,
+                    name: 'reset',
+                    url: null,
+                    isExternal: false,
+                    message: 'Reseting sub-process',
+                });
+            }
+            this.#destroyed();
+            return;
+        }
+        this.#destroyed();
+    }
+    getUndeadPid() {
+        return [...this.#undeadPid];
+    }
+    kill() {
+        if (!this.#subProcess) {
+            return;
+        }
+        if (!this.#subProcess.killed) {
+            scraperLog('Kills(SIGKILL) child_process (%d) ', this.#subProcess.pid);
+            this.#subProcess.kill('SIGKILL');
+            return;
+        }
+        scraperLog('child_process(%d) is already killed', this.#subProcess.pid);
+    }
+    start(url, options, isSkip, interval) {
+        if (this.#state === 'running') {
+            throw new Error(`Sub Routine (PID: ${this.#subProcess?.pid}) is already running`);
+        }
+        void this.#scrape(url, options, isSkip, interval);
+    }
+    #createSubProcess(url) {
+        const subProcess = childProcess.fork(SUB_PROCESS_PATH, {
+            detached: false,
+        });
+        subProcess.on('message', (action) => {
+            if (isType(action, scraperEvent.changePhase)) {
+                void this.emit('changePhase', action.payload);
+            }
+            if (isType(action, scraperEvent.ignoreAndSkip)) {
+                this.#finished();
+            }
+            if (isType(action, scraperEvent.scrapeEnd)) {
+                this.#finished();
+            }
+            if (isType(action, scraperEvent.destroyed)) {
+                this.#destroyed();
+            }
+            if (isType(action, scraperEvent.ignoreAndSkip) ||
+                isType(action, scraperEvent.resourceResponse) ||
+                isType(action, scraperEvent.scrapeEnd) ||
+                isType(action, scraperEvent.destroyed)) {
+                void this.emit('scrapeEvent', action);
+            }
+            if (isType(action, scraperEvent.error)) {
+                const error = new Error(action.payload.error.message);
+                error.name = action.payload.error.name;
+                error.stack = action.payload.error.stack;
+                const _action = {
+                    type: action.type,
+                    payload: {
+                        pid: action.payload.pid,
+                        url: action.payload.url,
+                        shutdown: action.payload.shutdown,
+                        error,
+                    },
+                };
+                void this.emit('scrapeEvent', _action);
+            }
+        });
+        subProcess.on('disconnect', () => {
+            if (subProcess.killed) {
+                scraperLog('child_process(%d) is disconnected and killed', subProcess.pid);
+                return;
+            }
+            scraperLog('child_process(%d) is disconnected but not killed', subProcess.pid);
+            scraperLog('Retries to kill(SIGTERM) child_process(%d)', subProcess.pid);
+            subProcess.kill('SIGTERM');
+            void this.emit('changePhase', {
+                pid: subProcess.pid,
+                name: 'disconnect',
+                url: null,
+                isExternal: false,
+                message: 'Disconnecting sub-process',
+            });
+        });
+        subProcess.on('error', (e) => {
+            void this.emit('error', {
+                pid: subProcess.pid,
+                url,
+                shutdown: true,
+                error: e instanceof Error ? e : new Error(`${e}`),
+            });
+        });
+        return subProcess;
+    }
+    #destroyed() {
+        if (this.#subProcess && !this.#subProcess.killed && this.#subProcess.pid) {
+            scraperLog('Add child_process(%d) to the undead PID list', this.#subProcess.pid);
+            this.#undeadPid.add(this.#subProcess.pid);
+        }
+        this.#subProcess = null;
+        this.#scrapedTimes = 0;
+        this.#finally();
+    }
+    #finally() {
+        this.#state = 'waiting';
+    }
+    #finished() {
+        this.#scrapedTimes += 1;
+        if (this.#scrapedTimes >= this.#resetTime) {
+            this.destory();
+        }
+        else {
+            this.#finally();
+        }
+    }
+    async #scrape(url, options, isSkip, interval) {
+        if (!this.#subProcess) {
+            this.#subProcess = this.#createSubProcess(url);
+            void this.emit('changePhase', {
+                pid: this.#subProcess.pid,
+                name: 'boot',
+                url,
+                isExternal: options.isExternal,
+                message: 'Booting sub-process',
+            });
+        }
+        this.#state = 'running';
+        interval = Math.max(interval, 0);
+        if (interval) {
+            await delay(interval);
+        }
+        this.#subProcess.send(subProcessEvent.start({
+            url,
+            isExternal: options.isExternal,
+            isGettingImages: options.isGettingImages,
+            excludeKeywords: options.excludeKeywords,
+            executablePath: options.executablePath,
+            disableQueries: options.disableQueries ?? false,
+            isSkip,
+            isTitleOnly: options.isTitleOnly,
+            screenshot: options.screenshot,
+        }));
+    }
+}

package/dist/sub-process.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/sub-process.js

@@ -0,0 +1,67 @@
+// Run on child_process
+import { isType } from 'typescript-fsa';
+import { scraperLog } from './debug.js';
+import { scraperEvent, subProcessEvent } from './events.js';
+import Scraper from './scraper.js';
+const log = scraperLog.extend(`${process.pid}`);
+process.title = 'beholder';
+const scraper = new Scraper();
+scraper.on('resourceResponse', (context) => {
+    if (process.connected) {
+        process.send(scraperEvent.resourceResponse(context));
+    }
+});
+scraper.on('ignoreAndSkip', (context) => {
+    if (process.connected) {
+        process.send(scraperEvent.ignoreAndSkip(context));
+    }
+});
+scraper.on('scrapeEnd', (context) => {
+    if (process.connected) {
+        process.send(scraperEvent.scrapeEnd(context));
+    }
+});
+scraper.on('error', (context) => {
+    if (process.connected) {
+        const _context = {
+            ...context,
+            error: {
+                name: context.error.name,
+                message: context.error.message,
+                stack: context.error.stack,
+            },
+        };
+        process.send(scraperEvent.error(_context));
+    }
+});
+scraper.on('changePhase', (context) => {
+    if (process.connected) {
+        process.send(scraperEvent.changePhase(context));
+    }
+});
+process.on('message', async (action) => {
+    if (isType(action, subProcessEvent.start)) {
+        void scraper.scrapeStart(action.payload.url, {
+            isExternal: action.payload.isExternal,
+            isGettingImages: action.payload.isGettingImages,
+            excludeKeywords: action.payload.excludeKeywords,
+            executablePath: action.payload.executablePath,
+            disableQueries: action.payload.disableQueries,
+            isTitleOnly: action.payload.isTitleOnly,
+            screenshot: action.payload.screenshot,
+        }, action.payload.isSkip);
+    }
+    if (isType(action, subProcessEvent.destroy)) {
+        await scraper.destroy(false);
+    }
+});
+scraper.on('destroyed', (context) => {
+    if (process.connected) {
+        process.send(scraperEvent.destroyed(context));
+        log('disconnects process');
+        process.disconnect();
+    }
+});
+process.on('disconnect', () => {
+    log('Process is disconnected');
+});

package/dist/types.d.ts

@@ -0,0 +1,271 @@
+import type { Action } from 'typescript-fsa';
+export type ScrapeEvent = {
+    pid: number | undefined;
+    url: ExURL;
+};
+export type ScrapeErrorEvent = ScrapeEvent & {
+    shutdown: boolean;
+    error: {
+        name: string;
+        message: string;
+        stack?: string;
+    };
+};
+export type ScrapeEventTypes = {
+    ignoreAndSkip: ScrapeEvent & {
+        reason: {
+            matchedText: string;
+            excludeKeywords: string[];
+        };
+    };
+    resourceResponse: ScrapeEvent & {
+        log: NetworkLog;
+        resource: Omit<Resource, 'uid'>;
+    };
+    scrapeEnd: ScrapeEvent & {
+        timestamp: number;
+        result: PageData;
+    };
+    destroyed: Omit<ScrapeEvent, 'url'>;
+    error: ScrapeErrorEvent;
+    changePhase: ChangePhaseEvent;
+};
+export type ChangePhaseEvent = {
+    pid: number;
+    name: 'scrapeStart' | 'launchBrowser' | 'touchHead' | 'touchHeadTimeout' | 'newPage' | 'openPage' | 'loadDOMContent' | 'waitNetworkIdleZero' | 'getHTML' | 'setViewport' | 'scrollToBottom' | 'getImages' | 'getAnchors' | 'getMeta' | 'ignoreAndSkip' | 'scrapeEnd' | 'beforeDestroy' | 'destroyed';
+    url: ExURL | null;
+    isExternal: boolean;
+    message: string;
+};
+export type AnyScrapeEvent = ScrapeEventTypes[keyof ScrapeEventTypes];
+export type SubProcessEventTypes = {
+    start: {
+        url: ExURL;
+        isExternal: boolean;
+        isGettingImages: boolean;
+        excludeKeywords: string[];
+        executablePath: string | null;
+        isSkip: boolean;
+        isTitleOnly: boolean;
+        screenshot: string | null;
+    } & Required<ParseURLOptions>;
+    destroy: void;
+};
+export type SubProcessEvent = {
+    pid: number | undefined;
+};
+export type SubProcessChangeEvent = ChangePhaseEvent | {
+    pid: number | undefined;
+    name: 'reset' | 'boot' | 'disconnect';
+    url: ExURL | null;
+    isExternal: boolean;
+    message: string;
+};
+export type SubProcessRunnerEventTypes = {
+    reset: SubProcessEvent;
+    scrapeEvent: Action<AnyScrapeEvent>;
+    changePhase: SubProcessChangeEvent;
+    error: ScrapeErrorEvent;
+};
+export type ExURL = {
+    /**
+     * Full URL (optimized)
+     */
+    href: string;
+    /**
+     * Full URL that before parse
+     */
+    _originUrlString: string;
+    /**
+     * Full URL without hash
+     */
+    withoutHash: string;
+    /**
+     * Full URL without hash and Authentication
+     */
+    withoutHashAndAuth: string;
+    /**
+     * Protocol or URI scheme (includes ":")
+     * - case-insensitive
+     */
+    protocol: string;
+    /**
+     * Whether protocol is HTTP or HTTPS
+     */
+    isHTTP: boolean;
+    /**
+     * Whether protocol is HTTPS
+     */
+    isSecure: boolean;
+    /**
+     * User name of authentication
+     */
+    username: string | null;
+    /**
+     * Password of authentication
+     */
+    password: string | null;
+    /**
+     * Host name
+     *
+     * - case-insensitive
+     * - encode non-ASCII characters
+     * - without port number
+     */
+    hostname: string;
+    /**
+     * Port number
+     */
+    port: string | null;
+    /**
+     * Path part
+     *
+     * It is only `/` if pathname is empty
+     *
+     * - case-sensitive
+     */
+    pathname: string | null;
+    /**
+     * Array of path
+     */
+    paths: string[];
+    /**
+     * Depth of paths
+     */
+    depth: number;
+    /**
+     * Directory name of paths
+     *
+     * It is null if it is `/` only
+     */
+    dirname: string | null;
+    /**
+     * Base name of paths (File name without file extension)
+     */
+    basename: string | null;
+    /**
+     * Whether index page (It's true if basename is null)
+     */
+    isIndex: boolean;
+    /**
+     * File extension name (inclues ".")
+     */
+    extname: string | null;
+    /**
+     * Search query (without `?`)
+     *
+     * - case-sensitive
+     */
+    query: string | null;
+    /**
+     * Hash (includes `#`)
+     *
+     * - case-sensitive
+     */
+    hash: string | null;
+};
+export type ParseURLOptions = {
+    disableQueries?: boolean;
+};
+export type PageData = {
+    url: ExURL;
+    redirectPaths: string[];
+    isTarget: boolean;
+    isExternal: boolean;
+    status: number;
+    statusText: string;
+    contentType: string | null;
+    contentLength: number | null;
+    responseHeaders: Record<string, string | string[] | undefined> | null;
+    meta: Meta;
+    anchorList: AnchorData[];
+    imageList: ImageElement[];
+    html: string;
+    isSkipped: false;
+};
+export type Meta = {
+    lang?: string;
+    title: string;
+    description?: string;
+    keywords?: string;
+    noindex?: boolean;
+    nofollow?: boolean;
+    noarchive?: boolean;
+    canonical?: string;
+    alternate?: string;
+    'og:type'?: string;
+    'og:title'?: string;
+    'og:site_name'?: string;
+    'og:description'?: string;
+    'og:url'?: string;
+    'og:image'?: string;
+    'twitter:card'?: string;
+};
+export type AnchorData = {
+    /**
+     * Extracts the value of the `href` attribute from anchor element (`<a>` `<area>`)
+     */
+    href: ExURL;
+    /**
+     * The accessible name of the anchor element
+     */
+    textContent: string;
+};
+export type ImageElement = {
+    src: string;
+    currentSrc: string;
+    alt: string;
+    width: number;
+    height: number;
+    naturalWidth: number;
+    naturalHeight: number;
+    isLazy: boolean;
+    viewportWidth: number;
+    sourceCode: string;
+};
+export type NetworkLog = {
+    url: ExURL;
+    status: number | null;
+    contentLength: number;
+    contentType: string;
+    isError: boolean;
+    request: {
+        ts: number;
+        headers: Record<string, string>;
+        method: string;
+    };
+    response?: {
+        ts: number;
+        status: number;
+        statusText: string;
+        fromCache: boolean;
+        headers: Record<string, string>;
+    };
+};
+export type Resource = {
+    url: ExURL;
+    isExternal: boolean;
+    isError: boolean;
+    status: number | null;
+    statusText: string | null;
+    contentType: string | null;
+    contentLength: number | null;
+    compress: false | CompressType;
+    cdn: false | CDNType;
+    headers: Record<string, string | string[] | undefined> | null;
+};
+export type CompressType = 'gzip' | 'compress' | 'deflate' | 'br' | 'sdch' | 'vcdiff' | 'xdelta';
+export type CDNType = 'Amazon S3' | 'Amazon CloudFront' | 'IIJ' | 'Cloudflare' | 'Akamai';
+export type HTTPMethod = 'HEAD' | 'GET' | 'POST' | 'PATCH' | 'PUT' | 'DELETE' | 'OPTIONS';
+export type SkippedPageData = {
+    isSkipped: true;
+    url: ExURL;
+    matched: {
+        type: 'keyword';
+        text: string;
+        excludeKeywords: string[];
+    } | {
+        type: 'path';
+        excludes: string[];
+    };
+};

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/dist/utils.d.ts

@@ -0,0 +1,5 @@
+import type { CDNType, CompressType, ParseURLOptions, ExURL } from './types.js';
+export declare function parseUrl(url: string | ExURL, options?: ParseURLOptions): ExURL | null;
+export declare function isError(status: number): boolean;
+export declare function detectCompress(headers: Record<string, string | string[] | undefined>): false | CompressType;
+export declare function detectCDN(headers: Record<string, string | string[] | undefined>): false | CDNType;

package/dist/utils.js

@@ -0,0 +1,142 @@
+import Path from 'node:path';
+export function parseUrl(url, options) {
+    if (typeof url !== 'string') {
+        return url;
+    }
+    try {
+        const whatwgUrl = new URL(url);
+        const { protocol, username, password, hostname, port, pathname, search, hash } = whatwgUrl;
+        const isHTTP = /^https?:$/.test(protocol);
+        const isSecure = protocol === 'https';
+        if (!isHTTP) {
+            const href = `${protocol}${pathname}`;
+            return {
+                href,
+                _originUrlString: url,
+                withoutHash: href,
+                withoutHashAndAuth: href,
+                protocol,
+                isHTTP: false,
+                isSecure: false,
+                username: null,
+                password: null,
+                hostname: '',
+                port: null,
+                pathname,
+                paths: [],
+                depth: 0,
+                dirname: null,
+                basename: null,
+                isIndex: false,
+                extname: null,
+                query: null,
+                hash: null,
+            };
+        }
+        const path = pathname.replaceAll(/\/+/g, '/');
+        const paths = path.replace(/^\//, '').split('/');
+        const isNoName = path.endsWith('/');
+        const extname = isNoName ? null : Path.extname(path);
+        const basename = isNoName ? null : Path.basename(path, extname || '');
+        const dirname = path === '/' ? null : Path.dirname(path + (isNoName ? 'index' : ''));
+        let query = search ? search.replace('?', '') : null;
+        if (options?.disableQueries) {
+            query = null;
+        }
+        else if (query) {
+            const param = new URLSearchParams(query);
+            param.delete('PHPSESSID');
+            param.sort();
+            query = param.toString();
+        }
+        const newSearch = query ? `?${query}` : '';
+        const auth = username && password ? `${username}:${password}@` : '';
+        const host = hostname + (port ? `:${port}` : '');
+        const body = dirname ? `${path}${newSearch}` : newSearch ? `/${newSearch}` : '';
+        const withoutHash = `${protocol}//${auth}${host}${body}`;
+        const withoutHashAndAuth = `${protocol}//${host}${body}`;
+        const href = withoutHash + (hash && !body && !withoutHash.endsWith('/') ? `/${hash}` : hash);
+        return {
+            href,
+            _originUrlString: url,
+            withoutHash,
+            withoutHashAndAuth,
+            protocol,
+            isHTTP,
+            isSecure,
+            username: username || null,
+            password: password || null,
+            hostname,
+            port: port || null,
+            pathname: path || null,
+            paths,
+            depth: paths.length,
+            dirname: dirname || null,
+            basename: basename || null,
+            isIndex: !basename || basename.toLowerCase() === 'index',
+            extname: extname || null,
+            query,
+            hash: hash || null,
+        };
+    }
+    catch {
+        return null;
+    }
+}
+export function isError(status) {
+    return !(200 <= status && status < 400);
+}
+export function detectCompress(headers) {
+    const enc = 'content-encoding' in headers && typeof headers['content-encoding'] === 'string'
+        ? headers['content-encoding']
+        : '';
+    if (/gzip/i.test(enc)) {
+        return 'gzip';
+    }
+    if (/br/i.test(enc)) {
+        return 'br';
+    }
+    if (/compress/i.test(enc)) {
+        return 'compress';
+    }
+    if (/deflate/i.test(enc)) {
+        return 'deflate';
+    }
+    // cspell:disable-next
+    if (/sdch/i.test(enc)) {
+        // cspell:disable-next
+        return 'sdch';
+    }
+    // cspell:disable-next
+    if (/vcdiff/i.test(enc)) {
+        // cspell:disable-next
+        return 'vcdiff';
+    }
+    // cspell:disable-next
+    if (/xdelta/i.test(enc)) {
+        // cspell:disable-next
+        return 'xdelta';
+    }
+    return false;
+}
+export function detectCDN(headers) {
+    if ('X-Akamai-Transformed' in headers) {
+        return 'Akamai';
+    }
+    if ('x-amz-cf-pop' in headers) {
+        return 'Amazon CloudFront';
+    }
+    if ('X-IIJ-Cache' in headers) {
+        return 'IIJ';
+    }
+    if (typeof headers.server === 'string') {
+        if (/cloudflare/i.test(headers.server)) {
+            return 'Cloudflare';
+        }
+        if (/amazons3/i.test(headers.server)) {
+            return 'Amazon S3';
+        }
+        return false;
+    }
+    return false;
+}