@d-zero/beholder 0.1.29 → 2.0.1

package/dist/scraper.js

@@ -1,621 +0,0 @@
-var __runInitializers = (this && this.__runInitializers) || function (thisArg, initializers, value) {
-    var useValue = arguments.length > 2;
-    for (var i = 0; i < initializers.length; i++) {
-        value = useValue ? initializers[i].call(thisArg, value) : initializers[i].call(thisArg);
-    }
-    return useValue ? value : void 0;
-};
-var __esDecorate = (this && this.__esDecorate) || function (ctor, descriptorIn, decorators, contextIn, initializers, extraInitializers) {
-    function accept(f) { if (f !== void 0 && typeof f !== "function") throw new TypeError("Function expected"); return f; }
-    var kind = contextIn.kind, key = kind === "getter" ? "get" : kind === "setter" ? "set" : "value";
-    var target = !descriptorIn && ctor ? contextIn["static"] ? ctor : ctor.prototype : null;
-    var descriptor = descriptorIn || (target ? Object.getOwnPropertyDescriptor(target, contextIn.name) : {});
-    var _, done = false;
-    for (var i = decorators.length - 1; i >= 0; i--) {
-        var context = {};
-        for (var p in contextIn) context[p] = p === "access" ? {} : contextIn[p];
-        for (var p in contextIn.access) context.access[p] = contextIn.access[p];
-        context.addInitializer = function (f) { if (done) throw new TypeError("Cannot add initializers after decoration has completed"); extraInitializers.push(accept(f || null)); };
-        var result = (0, decorators[i])(kind === "accessor" ? { get: descriptor.get, set: descriptor.set } : descriptor[key], context);
-        if (kind === "accessor") {
-            if (result === void 0) continue;
-            if (result === null || typeof result !== "object") throw new TypeError("Object expected");
-            if (_ = accept(result.get)) descriptor.get = _;
-            if (_ = accept(result.set)) descriptor.set = _;
-            if (_ = accept(result.init)) initializers.unshift(_);
-        }
-        else if (_ = accept(result)) {
-            if (kind === "field") initializers.unshift(_);
-            else descriptor[key] = _;
-        }
-    }
-    if (target) Object.defineProperty(target, contextIn.name, descriptor);
-    done = true;
-};
-var __setFunctionName = (this && this.__setFunctionName) || function (f, name, prefix) {
-    if (typeof name === "symbol") name = name.description ? "[".concat(name.description, "]") : "";
-    return Object.defineProperty(f, "name", { configurable: true, value: prefix ? "".concat(prefix, " ", name) : name });
-};
-import { beforePageScan } from '@d-zero/puppeteer-page-scan';
-import { parseUrl } from '@d-zero/shared/parse-url';
-import { retry } from '@d-zero/shared/retry';
-import { TypedAwaitEventEmitter } from '@d-zero/shared/typed-await-event-emitter';
-import { launch } from 'puppeteer';
-import { resourceLog, scraperLog } from './debug.js';
-import { getAnchorList, getImageList, getMeta } from './dom-evaluation.js';
-import { fetchDestination } from './fetch-destination.js';
-import { keywordCheck } from './keyword-check.js';
-import { detectCDN, detectCompress, isError } from './utils.js';
-const pid = `${process.pid}`;
-const log = scraperLog.extend(pid);
-const rLog = resourceLog.extend(pid);
-const LAUNCH_BROWSER_TIMEOUT = 1000 * 30;
-let Scraper = (() => {
-    let _classSuper = TypedAwaitEventEmitter;
-    let _instanceExtraInitializers = [];
-    let _private_bootBrowser_decorators;
-    let _private_bootBrowser_descriptor;
-    let _private_createPage_decorators;
-    let _private_createPage_descriptor;
-    let _private_fetchData_decorators;
-    let _private_fetchData_descriptor;
-    let _private_fetchHead_decorators;
-    let _private_fetchHead_descriptor;
-    let _private_fetchImages_decorators;
-    let _private_fetchImages_descriptor;
-    return class Scraper extends _classSuper {
-        static {
-            const _metadata = typeof Symbol === "function" && Symbol.metadata ? Object.create(_classSuper[Symbol.metadata] ?? null) : void 0;
-            _private_bootBrowser_decorators = [retry()];
-            _private_createPage_decorators = [retry()];
-            _private_fetchData_decorators = [retry({
-                    timeout: 1 * 60 * 1000, // 1sec,
-                })];
-            _private_fetchHead_decorators = [retry()];
-            _private_fetchImages_decorators = [retry({
-                    timeout: 5 * 60 * 1000, // 5sec
-                    fallback: [],
-                })];
-            __esDecorate(this, _private_bootBrowser_descriptor = { value: __setFunctionName(async function (isExternal, executablePath, headless) {
-                    if (!this.#browser) {
-                        void this.emit('changePhase', {
-                            pid: process.pid,
-                            name: 'launchBrowser',
-                            url: this.#url,
-                            isExternal,
-                            message: executablePath || '(executablePath is default)',
-                        });
-                        const browser = await launch({
-                            headless,
-                            timeout: LAUNCH_BROWSER_TIMEOUT,
-                            executablePath: executablePath ?? undefined,
-                            args: [
-                                // TODO: Optional lang
-                                '--lang=ja',
-                                '--no-zygote',
-                                '--ignore-certificate-errors',
-                            ],
-                        }).catch((error) => {
-                            if (error instanceof Error) {
-                                return error;
-                            }
-                            throw error;
-                        });
-                        if (browser instanceof Error) {
-                            void this.emit('error', {
-                                pid: process.pid,
-                                url: this.#url,
-                                shutdown: false,
-                                error: browser,
-                            });
-                            throw browser;
-                        }
-                        this.#browser = browser;
-                    }
-                    else if (!this.#browser.isConnected()) {
-                        await this.#browser.close();
-                    }
-                    return this.#browser;
-                }, "#bootBrowser") }, _private_bootBrowser_decorators, { kind: "method", name: "#bootBrowser", static: false, private: true, access: { has: obj => #bootBrowser in obj, get: obj => obj.#bootBrowser }, metadata: _metadata }, null, _instanceExtraInitializers);
-            __esDecorate(this, _private_createPage_descriptor = { value: __setFunctionName(async function (isExternal, executablePath, headless) {
-                    const browser = await this.#bootBrowser(isExternal, executablePath, headless);
-                    void this.emit('changePhase', {
-                        pid: process.pid,
-                        name: 'newPage',
-                        url: this.#url,
-                        isExternal,
-                        message: '',
-                    });
-                    const page = await browser.newPage();
-                    page.setDefaultNavigationTimeout(0);
-                    await page.setUserAgent('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36');
-                    await page.setExtraHTTPHeaders({
-                        // TODO: Optional lang
-                        'Accept-Language': 'ja-JP',
-                    });
-                    return page;
-                }, "#createPage") }, _private_createPage_decorators, { kind: "method", name: "#createPage", static: false, private: true, access: { has: obj => #createPage in obj, get: obj => obj.#createPage }, metadata: _metadata }, null, _instanceExtraInitializers);
-            __esDecorate(this, _private_fetchData_descriptor = { value: __setFunctionName(async function (page, url, isExternal, isGettingImages, options) {
-                    const networkLogs = {};
-                    page.on('dialog', async (dialog) => {
-                        log(`Appear ${dialog.type()} dialog: ${dialog.message()}`);
-                        try {
-                            await dialog.accept();
-                        }
-                        catch (error) {
-                            log(`Error: ${error}`);
-                        }
-                        log(`Accept ${dialog.type()} dialog`);
-                    });
-                    if (!isExternal) {
-                        page.on('request', (request) => {
-                            const url = parseUrl(request.url(), options);
-                            networkLogs[request.url()] = {
-                                url,
-                                status: null,
-                                contentLength: 0,
-                                contentType: '',
-                                isError: false,
-                                request: {
-                                    ts: Date.now(),
-                                    headers: request.headers(),
-                                    method: request.method(),
-                                },
-                            };
-                        });
-                        const uniqueRes = new Set();
-                        page.on('response', (response) => {
-                            const resURL = parseUrl(response.url(), options);
-                            if (uniqueRes.has(resURL.withoutHash)) {
-                                return;
-                            }
-                            if (resURL.withoutHash === url.withoutHash) {
-                                return;
-                            }
-                            uniqueRes.add(resURL.withoutHash);
-                            const headers = response.headers();
-                            const status = response.status();
-                            const statusText = response.statusText();
-                            const contentType = headers['content-type']?.split(';')[0] || null;
-                            const contentLength = Number.parseInt(headers['content-length'] ?? '', 10) || null;
-                            const request = networkLogs[resURL.withoutHash];
-                            const log = {
-                                ...request,
-                                response: {
-                                    ts: Date.now(),
-                                    status,
-                                    statusText,
-                                    fromCache: response.fromCache(),
-                                    headers,
-                                },
-                                status,
-                                isError: isError(status),
-                                contentType: contentType || '',
-                                contentLength: contentLength || 0,
-                            };
-                            const referredLink = {
-                                url: resURL,
-                                isExternal: resURL.hostname !== url.hostname,
-                                isError: log.isError,
-                                status,
-                                statusText,
-                                contentType,
-                                contentLength,
-                                compress: detectCompress(headers),
-                                cdn: detectCDN(headers),
-                                headers: headers,
-                            };
-                            rLog('Fetched: %s', resURL.href);
-                            void this.emit('resourceResponse', {
-                                pid: process.pid,
-                                url,
-                                log,
-                                resource: referredLink,
-                            });
-                        });
-                    }
-                    void this.emit('changePhase', {
-                        pid: process.pid,
-                        name: 'openPage',
-                        url: this.#url,
-                        isExternal,
-                        message: '',
-                    });
-                    if (url.username && url.password) {
-                        await page.setExtraHTTPHeaders({
-                            Authorization: `Basic ${Buffer.from(`${url.username}:${url.password}`).toString('base64')}`,
-                        });
-                    }
-                    const res = await page.goto(url.withoutHashAndAuth);
-                    if (!res) {
-                        throw new Error('The method Page.goto returned null');
-                    }
-                    const destUrl = parseUrl(page.url(), options);
-                    const redirectPaths = res
-                        .request()
-                        .redirectChain()
-                        .map((req) => req.url());
-                    if (destUrl.withoutHash !== url.withoutHash) {
-                        redirectPaths.push(destUrl.withoutHash);
-                    }
-                    if (destUrl.hostname !== url.hostname) {
-                        isExternal = true;
-                    }
-                    const status = res.status();
-                    const statusText = res.statusText();
-                    const responseHeaders = res.headers();
-                    const contentType = responseHeaders['content-type']?.split(';')[0] || null;
-                    const _contentLength = Number.parseInt(responseHeaders['content-length'] ?? '');
-                    const contentLength = Number.isFinite(_contentLength) ? _contentLength : null;
-                    if (contentType !== 'text/html') {
-                        return {
-                            url,
-                            isTarget: false,
-                            isExternal,
-                            redirectPaths,
-                            status,
-                            statusText,
-                            contentType,
-                            contentLength,
-                            responseHeaders,
-                            meta: {
-                                title: '',
-                            },
-                            imageList: [],
-                            anchorList: [],
-                            html: '',
-                            isSkipped: false,
-                        };
-                    }
-                    void this.emit('changePhase', {
-                        pid: process.pid,
-                        name: 'loadDOMContent',
-                        url: this.#url,
-                        isExternal,
-                        message: '',
-                    });
-                    await page
-                        .waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 5000 })
-                        .catch(() => { });
-                    void this.emit('changePhase', {
-                        pid: process.pid,
-                        name: 'getHTML',
-                        url: this.#url,
-                        isExternal,
-                        message: '',
-                    });
-                    const { title, html } = await page.evaluate(() => {
-                        /* global document */
-                        return {
-                            title: document.title,
-                            html: document.documentElement.outerHTML,
-                        };
-                    });
-                    if (isExternal) {
-                        return {
-                            url,
-                            isTarget: false,
-                            isExternal,
-                            redirectPaths,
-                            status,
-                            statusText,
-                            contentType,
-                            contentLength,
-                            responseHeaders,
-                            meta: {
-                                title,
-                            },
-                            imageList: [],
-                            anchorList: [],
-                            html,
-                            isSkipped: false,
-                        };
-                    }
-                    void this.emit('changePhase', {
-                        pid: process.pid,
-                        name: 'waitNetworkIdleZero',
-                        url: this.#url,
-                        isExternal,
-                        message: '',
-                    });
-                    await page
-                        .waitForNavigation({ waitUntil: 'networkidle0', timeout: 5000 })
-                        .catch(() => { });
-                    void this.emit('changePhase', {
-                        pid: process.pid,
-                        name: 'getAnchors',
-                        url: this.#url,
-                        isExternal,
-                        message: '',
-                    });
-                    const anchorList = await getAnchorList(page, options);
-                    void this.emit('changePhase', {
-                        pid: process.pid,
-                        name: 'getMeta',
-                        url: this.#url,
-                        isExternal,
-                        message: '',
-                    });
-                    const meta = await getMeta(page);
-                    const imageList = isGettingImages ? await this.#fetchImages(page, isExternal) : [];
-                    return {
-                        url,
-                        isTarget: true,
-                        isExternal,
-                        redirectPaths,
-                        status,
-                        statusText,
-                        contentType,
-                        contentLength,
-                        responseHeaders,
-                        meta,
-                        anchorList,
-                        imageList,
-                        html,
-                        isSkipped: false,
-                    };
-                }, "#fetchData") }, _private_fetchData_decorators, { kind: "method", name: "#fetchData", static: false, private: true, access: { has: obj => #fetchData in obj, get: obj => obj.#fetchData }, metadata: _metadata }, null, _instanceExtraInitializers);
-            __esDecorate(this, _private_fetchHead_descriptor = { value: __setFunctionName(async function (url, isExternal) {
-                    return await fetchDestination(url, isExternal);
-                }, "#fetchHead") }, _private_fetchHead_decorators, { kind: "method", name: "#fetchHead", static: false, private: true, access: { has: obj => #fetchHead in obj, get: obj => obj.#fetchHead }, metadata: _metadata }, null, _instanceExtraInitializers);
-            __esDecorate(this, _private_fetchImages_descriptor = { value: __setFunctionName(async function (page, isExternal) {
-                    const url = this.#url.withoutHashAndAuth;
-                    const imageList = [];
-                    const devices = [
-                        { name: 'desktop', width: 1280 },
-                        { name: 'mobile', width: 320, resolution: 2 },
-                    ];
-                    for (const device of devices) {
-                        void this.emit('changePhase', {
-                            pid: process.pid,
-                            name: 'setViewport',
-                            url: this.#url,
-                            isExternal,
-                            message: device.name,
-                        });
-                        await beforePageScan(page, url, {
-                            name: device.name,
-                            width: device.width,
-                            resolution: device.resolution,
-                            timeout: 5000,
-                        });
-                        void this.emit('changePhase', {
-                            pid: process.pid,
-                            name: 'getImages',
-                            url: this.#url,
-                            isExternal,
-                            message: device.name,
-                        });
-                        const images = await getImageList(page, device.width);
-                        imageList.push(...images);
-                    }
-                    return imageList;
-                }, "#fetchImages") }, _private_fetchImages_decorators, { kind: "method", name: "#fetchImages", static: false, private: true, access: { has: obj => #fetchImages in obj, get: obj => obj.#fetchImages }, metadata: _metadata }, null, _instanceExtraInitializers);
-            if (_metadata) Object.defineProperty(this, Symbol.metadata, { enumerable: true, configurable: true, writable: true, value: _metadata });
-        }
-        #browser = (__runInitializers(this, _instanceExtraInitializers), null);
-        #url = null;
-        async destroy(isExternal) {
-            log('Scraper destroys self');
-            if (!this.#url) {
-                throw new Error('The instance is already destroyed.');
-            }
-            if (!this.#browser) {
-                void this.emit('destroyed', {
-                    pid: process.pid,
-                });
-                void this.emit('changePhase', {
-                    pid: process.pid,
-                    name: 'destroyed',
-                    url: this.#url,
-                    isExternal,
-                    message: '',
-                });
-                return;
-            }
-            while (!this.#browser.isConnected()) {
-                log('Browser closes all pages');
-                const pages = await this.#browser.pages();
-                for (const page of pages) {
-                    page.removeAllListeners();
-                    if (!page.isClosed) {
-                        await page.close();
-                    }
-                }
-                log('Browser closes self');
-                await this.#browser.close();
-                log('Browser disconnects');
-                await this.#browser.disconnect();
-            }
-            log('Scraper discards browser');
-            this.#browser = null;
-            void this.emit('destroyed', {
-                pid: process.pid,
-            });
-            void this.emit('changePhase', {
-                pid: process.pid,
-                name: 'destroyed',
-                url: this.#url,
-                isExternal,
-                message: '',
-            });
-        }
-        async scrapeStart(url, options, isSkip = false) {
-            const isExternal = options?.isExternal ?? false;
-            const isGettingImages = options?.isGettingImages ?? true;
-            const excludeKeywords = options?.excludeKeywords ?? [];
-            const executablePath = options?.executablePath ?? null;
-            const isTitleOnly = options?.isTitleOnly ?? false;
-            this.#url = url;
-            void this.emit('changePhase', {
-                pid: process.pid,
-                name: 'scrapeStart',
-                url: this.#url,
-                isExternal,
-                message: '',
-            });
-            if (isSkip) {
-                void this.emit('ignoreAndSkip', {
-                    pid: process.pid,
-                    url: this.#url,
-                    reason: {
-                        matchedText: this.#url.pathname || '',
-                        excludeKeywords,
-                    },
-                });
-                void this.emit('changePhase', {
-                    pid: process.pid,
-                    name: 'ignoreAndSkip',
-                    url: this.#url,
-                    isExternal,
-                    message: 'Matched: excluded path',
-                });
-                return;
-            }
-            if (!this.#url.isHTTP) {
-                const result = {
-                    url: this.#url,
-                    isTarget: false,
-                    isExternal,
-                    redirectPaths: [],
-                    status: -1,
-                    statusText: '__THIS_IS_NOT_HTTP_PROTOCOL__',
-                    contentType: null,
-                    contentLength: null,
-                    responseHeaders: {},
-                    meta: {
-                        title: '',
-                    },
-                    imageList: [],
-                    anchorList: [],
-                    html: '',
-                    isSkipped: false,
-                };
-                void this.emit('scrapeEnd', {
-                    pid: process.pid,
-                    url: this.#url,
-                    timestamp: Date.now(),
-                    result,
-                });
-                void this.emit('changePhase', {
-                    pid: process.pid,
-                    name: 'scrapeEnd',
-                    url: this.#url,
-                    isExternal,
-                    message: '',
-                });
-                return;
-            }
-            void this.emit('changePhase', {
-                pid: process.pid,
-                name: 'touchHead',
-                url: this.#url,
-                isExternal,
-                message: '',
-            });
-            let result = await this.#fetchHead(url, isExternal);
-            if (result instanceof Error) {
-                log('Error(FETCH_HEAD): %s', url.href);
-                void this.emit('error', {
-                    pid: process.pid,
-                    url: this.#url,
-                    shutdown: false,
-                    error: result,
-                });
-                result = null;
-            }
-            if (result && isTitleOnly) {
-                void this.emit('scrapeEnd', {
-                    pid: process.pid,
-                    url: this.#url,
-                    timestamp: Date.now(),
-                    result: {
-                        ...result,
-                        isTarget: false,
-                    },
-                });
-                return;
-            }
-            if (result === null || result.contentType === 'text/html') {
-                const headlessMode = url.isSecure ? true : 'shell';
-                const page = await this.#createPage(isExternal, executablePath, headlessMode);
-                result = await this.#fetchData(page, url, isExternal, isGettingImages, options).catch((error) => {
-                    if (error instanceof Error) {
-                        return error;
-                    }
-                    return new Error(error);
-                });
-                if (result instanceof Error) {
-                    log('Error(FETCH_DATA): %s', url.href);
-                    void this.emit('error', {
-                        pid: process.pid,
-                        url: this.#url,
-                        shutdown: true,
-                        error: result,
-                    });
-                    await this.destroy(isExternal);
-                    return;
-                }
-                page.removeAllListeners();
-                if (!page.isClosed) {
-                    await page.close();
-                }
-                if (!result.isSkipped) {
-                    const checkedKeyword = keywordCheck(result.html, excludeKeywords);
-                    if (checkedKeyword) {
-                        result = {
-                            url,
-                            isSkipped: true,
-                            matched: {
-                                type: 'keyword',
-                                text: checkedKeyword,
-                                excludeKeywords,
-                            },
-                        };
-                    }
-                }
-                if (result.isSkipped) {
-                    if (result.matched.type === 'path') {
-                        return;
-                    }
-                    void this.emit('ignoreAndSkip', {
-                        pid: process.pid,
-                        url: this.#url,
-                        reason: {
-                            matchedText: result.matched.text,
-                            excludeKeywords,
-                        },
-                    });
-                    void this.emit('changePhase', {
-                        pid: process.pid,
-                        name: 'ignoreAndSkip',
-                        url: this.#url,
-                        isExternal,
-                        message: `Matched: "${result.matched.text}"`,
-                    });
-                    return;
-                }
-            }
-            void this.emit('scrapeEnd', {
-                pid: process.pid,
-                url: this.#url,
-                timestamp: Date.now(),
-                result,
-            });
-            void this.emit('changePhase', {
-                pid: process.pid,
-                name: 'scrapeEnd',
-                url: this.#url,
-                isExternal,
-                message: '',
-            });
-            return result;
-        }
-        get #bootBrowser() { return _private_bootBrowser_descriptor.value; }
-        get #createPage() { return _private_createPage_descriptor.value; }
-        get #fetchData() { return _private_fetchData_descriptor.value; }
-        get #fetchHead() { return _private_fetchHead_descriptor.value; }
-        get #fetchImages() { return _private_fetchImages_descriptor.value; }
-    };
-})();
-export default Scraper;

package/dist/sub-process-runner.d.ts

@@ -1,12 +0,0 @@
-import type { ScraperOptions } from './scraper.js';
-import type { SubProcessRunnerEventTypes, ExURL } from './types.js';
-import { TypedAwaitEventEmitter } from '@d-zero/shared/typed-await-event-emitter';
-export default class SubProcessRunner extends TypedAwaitEventEmitter<SubProcessRunnerEventTypes> {
-    #private;
-    get state(): "waiting" | "running";
-    constructor(resetTime: number);
-    destory(): void;
-    getUndeadPid(): number[];
-    kill(): void;
-    start(url: ExURL, options: ScraperOptions, isSkip: boolean, interval: number): void;
-}