@d-zero/beholder 0.1.29 → 2.0.1

package/dist/sub-process-runner.js

@@ -1,180 +0,0 @@
-import childProcess from 'node:child_process';
-import path from 'node:path';
-import { delay } from '@d-zero/shared/delay';
-import { TypedAwaitEventEmitter } from '@d-zero/shared/typed-await-event-emitter';
-import { isType } from 'typescript-fsa';
-import { scraperLog } from './debug.js';
-import { scraperEvent, subProcessEvent } from './events.js';
-const __filename = new globalThis.URL(import.meta.url).pathname;
-const __dirname = path.dirname(__filename);
-const SUB_PROCESS_PATH = path.resolve(__dirname, 'sub-process');
-export default class SubProcessRunner extends TypedAwaitEventEmitter {
-    #resetTime;
-    #scrapedTimes = 0;
-    #state = 'waiting';
-    #subProcess = null;
-    #undeadPid = new Set();
-    get state() {
-        return this.#state;
-    }
-    constructor(resetTime) {
-        super();
-        this.#resetTime = resetTime;
-    }
-    destory() {
-        const pid = this.#subProcess?.pid;
-        if (this.#subProcess) {
-            scraperLog('Destroys child_process (%d)', pid);
-            this.#subProcess.send(subProcessEvent.destroy());
-            if (pid) {
-                void this.emit('reset', {
-                    pid,
-                });
-                void this.emit('changePhase', {
-                    pid,
-                    name: 'reset',
-                    url: null,
-                    isExternal: false,
-                    message: 'Reseting sub-process',
-                });
-            }
-            this.#destroyed();
-            return;
-        }
-        this.#destroyed();
-    }
-    getUndeadPid() {
-        return [...this.#undeadPid];
-    }
-    kill() {
-        if (!this.#subProcess) {
-            return;
-        }
-        if (!this.#subProcess.killed) {
-            scraperLog('Kills(SIGKILL) child_process (%d) ', this.#subProcess.pid);
-            this.#subProcess.kill('SIGKILL');
-            return;
-        }
-        scraperLog('child_process(%d) is already killed', this.#subProcess.pid);
-    }
-    start(url, options, isSkip, interval) {
-        if (this.#state === 'running') {
-            throw new Error(`Sub Routine (PID: ${this.#subProcess?.pid}) is already running`);
-        }
-        void this.#scrape(url, options, isSkip, interval);
-    }
-    #createSubProcess(url) {
-        const subProcess = childProcess.fork(SUB_PROCESS_PATH, {
-            detached: false,
-        });
-        subProcess.on('message', (action) => {
-            if (isType(action, scraperEvent.changePhase)) {
-                void this.emit('changePhase', action.payload);
-            }
-            if (isType(action, scraperEvent.ignoreAndSkip)) {
-                this.#finished();
-            }
-            if (isType(action, scraperEvent.scrapeEnd)) {
-                this.#finished();
-            }
-            if (isType(action, scraperEvent.destroyed)) {
-                this.#destroyed();
-            }
-            if (isType(action, scraperEvent.ignoreAndSkip) ||
-                isType(action, scraperEvent.resourceResponse) ||
-                isType(action, scraperEvent.scrapeEnd) ||
-                isType(action, scraperEvent.destroyed)) {
-                void this.emit('scrapeEvent', action);
-            }
-            if (isType(action, scraperEvent.error)) {
-                const error = new Error(action.payload.error.message);
-                error.name = action.payload.error.name;
-                error.stack = action.payload.error.stack;
-                const _action = {
-                    type: action.type,
-                    payload: {
-                        pid: action.payload.pid,
-                        url: action.payload.url,
-                        shutdown: action.payload.shutdown,
-                        error,
-                    },
-                };
-                void this.emit('scrapeEvent', _action);
-            }
-        });
-        subProcess.on('disconnect', () => {
-            if (subProcess.killed) {
-                scraperLog('child_process(%d) is disconnected and killed', subProcess.pid);
-                return;
-            }
-            scraperLog('child_process(%d) is disconnected but not killed', subProcess.pid);
-            scraperLog('Retries to kill(SIGTERM) child_process(%d)', subProcess.pid);
-            subProcess.kill('SIGTERM');
-            void this.emit('changePhase', {
-                pid: subProcess.pid,
-                name: 'disconnect',
-                url: null,
-                isExternal: false,
-                message: 'Disconnecting sub-process',
-            });
-        });
-        subProcess.on('error', (e) => {
-            void this.emit('error', {
-                pid: subProcess.pid,
-                url,
-                shutdown: true,
-                error: e instanceof Error ? e : new Error(`${e}`),
-            });
-        });
-        return subProcess;
-    }
-    #destroyed() {
-        if (this.#subProcess && !this.#subProcess.killed && this.#subProcess.pid) {
-            scraperLog('Add child_process(%d) to the undead PID list', this.#subProcess.pid);
-            this.#undeadPid.add(this.#subProcess.pid);
-        }
-        this.#subProcess = null;
-        this.#scrapedTimes = 0;
-        this.#finally();
-    }
-    #finally() {
-        this.#state = 'waiting';
-    }
-    #finished() {
-        this.#scrapedTimes += 1;
-        if (this.#scrapedTimes >= this.#resetTime) {
-            this.destory();
-        }
-        else {
-            this.#finally();
-        }
-    }
-    async #scrape(url, options, isSkip, interval) {
-        if (!this.#subProcess) {
-            this.#subProcess = this.#createSubProcess(url);
-            void this.emit('changePhase', {
-                pid: this.#subProcess.pid,
-                name: 'boot',
-                url,
-                isExternal: options.isExternal,
-                message: 'Booting sub-process',
-            });
-        }
-        this.#state = 'running';
-        interval = Math.max(interval, 0);
-        if (interval) {
-            await delay(interval);
-        }
-        this.#subProcess.send(subProcessEvent.start({
-            url,
-            isExternal: options.isExternal,
-            isGettingImages: options.isGettingImages,
-            excludeKeywords: options.excludeKeywords,
-            executablePath: options.executablePath,
-            disableQueries: options.disableQueries ?? false,
-            isSkip,
-            isTitleOnly: options.isTitleOnly,
-            screenshot: options.screenshot,
-        }));
-    }
-}

package/dist/sub-process.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/sub-process.js

@@ -1,67 +0,0 @@
-// Run on child_process
-import { isType } from 'typescript-fsa';
-import { scraperLog } from './debug.js';
-import { scraperEvent, subProcessEvent } from './events.js';
-import Scraper from './scraper.js';
-const log = scraperLog.extend(`${process.pid}`);
-process.title = 'beholder';
-const scraper = new Scraper();
-scraper.on('resourceResponse', (context) => {
-    if (process.connected) {
-        process.send(scraperEvent.resourceResponse(context));
-    }
-});
-scraper.on('ignoreAndSkip', (context) => {
-    if (process.connected) {
-        process.send(scraperEvent.ignoreAndSkip(context));
-    }
-});
-scraper.on('scrapeEnd', (context) => {
-    if (process.connected) {
-        process.send(scraperEvent.scrapeEnd(context));
-    }
-});
-scraper.on('error', (context) => {
-    if (process.connected) {
-        const _context = {
-            ...context,
-            error: {
-                name: context.error.name,
-                message: context.error.message,
-                stack: context.error.stack,
-            },
-        };
-        process.send(scraperEvent.error(_context));
-    }
-});
-scraper.on('changePhase', (context) => {
-    if (process.connected) {
-        process.send(scraperEvent.changePhase(context));
-    }
-});
-process.on('message', async (action) => {
-    if (isType(action, subProcessEvent.start)) {
-        void scraper.scrapeStart(action.payload.url, {
-            isExternal: action.payload.isExternal,
-            isGettingImages: action.payload.isGettingImages,
-            excludeKeywords: action.payload.excludeKeywords,
-            executablePath: action.payload.executablePath,
-            disableQueries: action.payload.disableQueries,
-            isTitleOnly: action.payload.isTitleOnly,
-            screenshot: action.payload.screenshot,
-        }, action.payload.isSkip);
-    }
-    if (isType(action, subProcessEvent.destroy)) {
-        await scraper.destroy(false);
-    }
-});
-scraper.on('destroyed', (context) => {
-    if (process.connected) {
-        process.send(scraperEvent.destroyed(context));
-        log('disconnects process');
-        process.disconnect();
-    }
-});
-process.on('disconnect', () => {
-    log('Process is disconnected');
-});

package/dist/types.d.ts

@@ -1,271 +0,0 @@
-import type { Action } from 'typescript-fsa';
-export type ScrapeEvent = {
-    pid: number | undefined;
-    url: ExURL;
-};
-export type ScrapeErrorEvent = ScrapeEvent & {
-    shutdown: boolean;
-    error: {
-        name: string;
-        message: string;
-        stack?: string;
-    };
-};
-export type ScrapeEventTypes = {
-    ignoreAndSkip: ScrapeEvent & {
-        reason: {
-            matchedText: string;
-            excludeKeywords: string[];
-        };
-    };
-    resourceResponse: ScrapeEvent & {
-        log: NetworkLog;
-        resource: Omit<Resource, 'uid'>;
-    };
-    scrapeEnd: ScrapeEvent & {
-        timestamp: number;
-        result: PageData;
-    };
-    destroyed: Omit<ScrapeEvent, 'url'>;
-    error: ScrapeErrorEvent;
-    changePhase: ChangePhaseEvent;
-};
-export type ChangePhaseEvent = {
-    pid: number;
-    name: 'scrapeStart' | 'launchBrowser' | 'touchHead' | 'touchHeadTimeout' | 'newPage' | 'openPage' | 'loadDOMContent' | 'waitNetworkIdleZero' | 'getHTML' | 'setViewport' | 'scrollToBottom' | 'getImages' | 'getAnchors' | 'getMeta' | 'ignoreAndSkip' | 'scrapeEnd' | 'beforeDestroy' | 'destroyed';
-    url: ExURL | null;
-    isExternal: boolean;
-    message: string;
-};
-export type AnyScrapeEvent = ScrapeEventTypes[keyof ScrapeEventTypes];
-export type SubProcessEventTypes = {
-    start: {
-        url: ExURL;
-        isExternal: boolean;
-        isGettingImages: boolean;
-        excludeKeywords: string[];
-        executablePath: string | null;
-        isSkip: boolean;
-        isTitleOnly: boolean;
-        screenshot: string | null;
-    } & Required<ParseURLOptions>;
-    destroy: void;
-};
-export type SubProcessEvent = {
-    pid: number | undefined;
-};
-export type SubProcessChangeEvent = ChangePhaseEvent | {
-    pid: number | undefined;
-    name: 'reset' | 'boot' | 'disconnect';
-    url: ExURL | null;
-    isExternal: boolean;
-    message: string;
-};
-export type SubProcessRunnerEventTypes = {
-    reset: SubProcessEvent;
-    scrapeEvent: Action<AnyScrapeEvent>;
-    changePhase: SubProcessChangeEvent;
-    error: ScrapeErrorEvent;
-};
-export type ExURL = {
-    /**
-     * Full URL (optimized)
-     */
-    href: string;
-    /**
-     * Full URL that before parse
-     */
-    _originUrlString: string;
-    /**
-     * Full URL without hash
-     */
-    withoutHash: string;
-    /**
-     * Full URL without hash and Authentication
-     */
-    withoutHashAndAuth: string;
-    /**
-     * Protocol or URI scheme (includes ":")
-     * - case-insensitive
-     */
-    protocol: string;
-    /**
-     * Whether protocol is HTTP or HTTPS
-     */
-    isHTTP: boolean;
-    /**
-     * Whether protocol is HTTPS
-     */
-    isSecure: boolean;
-    /**
-     * User name of authentication
-     */
-    username: string | null;
-    /**
-     * Password of authentication
-     */
-    password: string | null;
-    /**
-     * Host name
-     *
-     * - case-insensitive
-     * - encode non-ASCII characters
-     * - without port number
-     */
-    hostname: string;
-    /**
-     * Port number
-     */
-    port: string | null;
-    /**
-     * Path part
-     *
-     * It is only `/` if pathname is empty
-     *
-     * - case-sensitive
-     */
-    pathname: string | null;
-    /**
-     * Array of path
-     */
-    paths: string[];
-    /**
-     * Depth of paths
-     */
-    depth: number;
-    /**
-     * Directory name of paths
-     *
-     * It is null if it is `/` only
-     */
-    dirname: string | null;
-    /**
-     * Base name of paths (File name without file extension)
-     */
-    basename: string | null;
-    /**
-     * Whether index page (It's true if basename is null)
-     */
-    isIndex: boolean;
-    /**
-     * File extension name (inclues ".")
-     */
-    extname: string | null;
-    /**
-     * Search query (without `?`)
-     *
-     * - case-sensitive
-     */
-    query: string | null;
-    /**
-     * Hash (includes `#`)
-     *
-     * - case-sensitive
-     */
-    hash: string | null;
-};
-export type ParseURLOptions = {
-    disableQueries?: boolean;
-};
-export type PageData = {
-    url: ExURL;
-    redirectPaths: string[];
-    isTarget: boolean;
-    isExternal: boolean;
-    status: number;
-    statusText: string;
-    contentType: string | null;
-    contentLength: number | null;
-    responseHeaders: Record<string, string | string[] | undefined> | null;
-    meta: Meta;
-    anchorList: AnchorData[];
-    imageList: ImageElement[];
-    html: string;
-    isSkipped: false;
-};
-export type Meta = {
-    lang?: string;
-    title: string;
-    description?: string;
-    keywords?: string;
-    noindex?: boolean;
-    nofollow?: boolean;
-    noarchive?: boolean;
-    canonical?: string;
-    alternate?: string;
-    'og:type'?: string;
-    'og:title'?: string;
-    'og:site_name'?: string;
-    'og:description'?: string;
-    'og:url'?: string;
-    'og:image'?: string;
-    'twitter:card'?: string;
-};
-export type AnchorData = {
-    /**
-     * Extracts the value of the `href` attribute from anchor element (`<a>` `<area>`)
-     */
-    href: ExURL;
-    /**
-     * The accessible name of the anchor element
-     */
-    textContent: string;
-};
-export type ImageElement = {
-    src: string;
-    currentSrc: string;
-    alt: string;
-    width: number;
-    height: number;
-    naturalWidth: number;
-    naturalHeight: number;
-    isLazy: boolean;
-    viewportWidth: number;
-    sourceCode: string;
-};
-export type NetworkLog = {
-    url: ExURL;
-    status: number | null;
-    contentLength: number;
-    contentType: string;
-    isError: boolean;
-    request: {
-        ts: number;
-        headers: Record<string, string>;
-        method: string;
-    };
-    response?: {
-        ts: number;
-        status: number;
-        statusText: string;
-        fromCache: boolean;
-        headers: Record<string, string>;
-    };
-};
-export type Resource = {
-    url: ExURL;
-    isExternal: boolean;
-    isError: boolean;
-    status: number | null;
-    statusText: string | null;
-    contentType: string | null;
-    contentLength: number | null;
-    compress: false | CompressType;
-    cdn: false | CDNType;
-    headers: Record<string, string | string[] | undefined> | null;
-};
-export type CompressType = 'gzip' | 'compress' | 'deflate' | 'br' | 'sdch' | 'vcdiff' | 'xdelta';
-export type CDNType = 'Amazon S3' | 'Amazon CloudFront' | 'IIJ' | 'Cloudflare' | 'Akamai';
-export type HTTPMethod = 'HEAD' | 'GET' | 'POST' | 'PATCH' | 'PUT' | 'DELETE' | 'OPTIONS';
-export type SkippedPageData = {
-    isSkipped: true;
-    url: ExURL;
-    matched: {
-        type: 'keyword';
-        text: string;
-        excludeKeywords: string[];
-    } | {
-        type: 'path';
-        excludes: string[];
-    };
-};

package/dist/types.js

@@ -1 +0,0 @@
-export {};

package/dist/utils.d.ts

@@ -1,16 +0,0 @@
-import type { CDNType, CompressType } from './types.js';
-/**
- *
- * @param status
- */
-export declare function isError(status: number): boolean;
-/**
- *
- * @param headers
- */
-export declare function detectCompress(headers: Record<string, string | string[] | undefined>): false | CompressType;
-/**
- *
- * @param headers
- */
-export declare function detectCDN(headers: Record<string, string | string[] | undefined>): false | CDNType;

package/dist/utils.js

@@ -1,69 +0,0 @@
-/**
- *
- * @param status
- */
-export function isError(status) {
-    return !(200 <= status && status < 400);
-}
-/**
- *
- * @param headers
- */
-export function detectCompress(headers) {
-    const enc = 'content-encoding' in headers && typeof headers['content-encoding'] === 'string'
-        ? headers['content-encoding']
-        : '';
-    if (/gzip/i.test(enc)) {
-        return 'gzip';
-    }
-    if (/br/i.test(enc)) {
-        return 'br';
-    }
-    if (/compress/i.test(enc)) {
-        return 'compress';
-    }
-    if (/deflate/i.test(enc)) {
-        return 'deflate';
-    }
-    // cspell:disable-next
-    if (/sdch/i.test(enc)) {
-        // cspell:disable-next
-        return 'sdch';
-    }
-    // cspell:disable-next
-    if (/vcdiff/i.test(enc)) {
-        // cspell:disable-next
-        return 'vcdiff';
-    }
-    // cspell:disable-next
-    if (/xdelta/i.test(enc)) {
-        // cspell:disable-next
-        return 'xdelta';
-    }
-    return false;
-}
-/**
- *
- * @param headers
- */
-export function detectCDN(headers) {
-    if ('X-Akamai-Transformed' in headers) {
-        return 'Akamai';
-    }
-    if ('x-amz-cf-pop' in headers) {
-        return 'Amazon CloudFront';
-    }
-    if ('X-IIJ-Cache' in headers) {
-        return 'IIJ';
-    }
-    if (typeof headers.server === 'string') {
-        if (/cloudflare/i.test(headers.server)) {
-            return 'Cloudflare';
-        }
-        if (/amazons3/i.test(headers.server)) {
-            return 'Amazon S3';
-        }
-        return false;
-    }
-    return false;
-}

package/src/events.ts

@@ -1,21 +0,0 @@
-import type { ScrapeEventTypes, SubProcessEventTypes } from './types.js';
-
-import { actionCreatorFactory } from 'typescript-fsa';
-
-const scraperEventCreator = actionCreatorFactory('@@scraper');
-const subProcessEventCreator = actionCreatorFactory('@@sub-process');
-
-export const subProcessEvent = {
-	start: subProcessEventCreator<SubProcessEventTypes['start']>('start'),
-	destroy: subProcessEventCreator<SubProcessEventTypes['destroy']>('destroy'),
-};
-
-export const scraperEvent = {
-	ignoreAndSkip: scraperEventCreator<ScrapeEventTypes['ignoreAndSkip']>('ignoreAndSkip'),
-	resourceResponse:
-		scraperEventCreator<ScrapeEventTypes['resourceResponse']>('resourceResponse'),
-	scrapeEnd: scraperEventCreator<ScrapeEventTypes['scrapeEnd']>('scrapeEnd'),
-	destroyed: scraperEventCreator<ScrapeEventTypes['destroyed']>('destroyed'),
-	error: scraperEventCreator<ScrapeEventTypes['error']>('error'),
-	changePhase: scraperEventCreator<ScrapeEventTypes['changePhase']>('changePhase'),
-};