@d-zero/beholder 0.1.29 → 2.0.1

package/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2024 D-ZERO Co., Ltd.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

package/dist/debug.d.ts

@@ -1,6 +0,0 @@
-import debug from 'debug';
-export declare const log: debug.Debugger;
-export declare const scraperLog: debug.Debugger;
-export declare const resourceLog: debug.Debugger;
-export declare const domLog: debug.Debugger;
-export declare const domDetailsLog: debug.Debugger;

package/dist/debug.js

@@ -1,6 +0,0 @@
-import debug from 'debug';
-export const log = debug('Beholder');
-export const scraperLog = log.extend('Scraper');
-export const resourceLog = scraperLog.extend('Resource');
-export const domLog = scraperLog.extend('DOM');
-export const domDetailsLog = domLog.extend('Details');

package/dist/dom-evaluation.d.ts

@@ -1,51 +0,0 @@
-import type { AnchorData, ImageElement, ParseURLOptions } from './types.js';
-import type { ElementHandle, Page } from 'puppeteer';
-/**
- *
- * @param $el
- * @param propName
- * @param fallback
- */
-export declare function getProp<T>($el: ElementHandle<Element>, propName: string, fallback: T): Promise<T>;
-/**
- *
- * @param page
- * @param selector
- * @param propName
- * @param fallback
- */
-export declare function getPropBySelector<T>(page: Page, selector: string, propName: string, fallback: T): Promise<T>;
-/**
- *
- * @param page
- * @param viewportWidth
- */
-export declare function getImageList(page: Page, viewportWidth: number): Promise<ImageElement[]>;
-/**
- *
- * @param page
- * @param options
- */
-export declare function getAnchorList(page: Page, options?: ParseURLOptions): Promise<AnchorData[]>;
-/**
- *
- * @param page
- */
-export declare function getMeta(page: Page): Promise<{
-    title: string;
-    lang: string;
-    description: string;
-    keywords: string;
-    noindex: boolean;
-    nofollow: boolean;
-    noarchive: boolean;
-    canonical: string;
-    alternate: string;
-    'og:type': string;
-    'og:title': string;
-    'og:site_name': string;
-    'og:description': string;
-    'og:url': string;
-    'og:image': string;
-    'twitter:card': string;
-}>;

package/dist/dom-evaluation.js

@@ -1,147 +0,0 @@
-import { parseUrl } from '@d-zero/shared/parse-url';
-import { domDetailsLog, domLog } from './debug.js';
-const pid = `${process.pid}`;
-const log = domLog.extend(pid);
-const dLog = domDetailsLog.extend(pid);
-/**
- *
- * @param $el
- * @param propName
- * @param fallback
- */
-export async function getProp($el, propName, fallback) {
-    return Promise.race([
-        _getProp($el, propName, fallback),
-        new Promise((res) => setTimeout(() => res(fallback), 10 * 1000)),
-    ]);
-}
-/**
- *
- * @param $el
- * @param propName
- * @param fallback
- */
-async function _getProp($el, propName, fallback) {
-    try {
-        const prop = await $el.getProperty(propName);
-        if (!prop) {
-            return fallback;
-        }
-        const value = (await prop.jsonValue());
-        return value;
-    }
-    catch {
-        return fallback;
-    }
-}
-/**
- *
- * @param page
- * @param selector
- * @param propName
- * @param fallback
- */
-export async function getPropBySelector(page, selector, propName, fallback) {
-    const $el = await page.$(selector);
-    if (!$el) {
-        return fallback;
-    }
-    return getProp($el, propName, fallback);
-}
-/**
- *
- * @param page
- * @param viewportWidth
- */
-export async function getImageList(page, viewportWidth) {
-    log('Getting images (Viewport: %dpx)', viewportWidth);
-    const $images = await page.$$('img');
-    const imageList = [];
-    for (const $image of $images) {
-        const boundingBox = await $image.boundingBox();
-        const width = boundingBox?.width || 0;
-        const height = boundingBox?.height || 0;
-        const src = await getProp($image, 'src', '');
-        const currentSrc = await getProp($image, 'currentSrc', '');
-        const alt = await getProp($image, 'alt', '');
-        const naturalWidth = await getProp($image, 'naturalWidth', 0);
-        const naturalHeight = await getProp($image, 'naturalHeight', 0);
-        const loading = await getProp($image, 'loading', '');
-        const sourceCode = await getProp($image, 'outerHTML', '');
-        const isLazy = loading.toLowerCase().trim() === 'lazy';
-        imageList.push({
-            src,
-            currentSrc,
-            alt,
-            width,
-            height,
-            naturalWidth,
-            naturalHeight,
-            isLazy,
-            viewportWidth,
-            sourceCode,
-        });
-    }
-    log('Got %d images (Viewport: %dpx)', imageList.length, viewportWidth);
-    dLog('Images are: %O', imageList.map((i) => i.src));
-    return imageList;
-}
-/**
- *
- * @param page
- * @param options
- */
-export async function getAnchorList(page, options) {
-    log('Getting anchors');
-    const $anchors = await page.$$('a[href], area[href]');
-    const anchorList = [];
-    for (const $anchor of $anchors) {
-        const $href = await getProp($anchor, 'href', '');
-        const hrefVal = $href.toString();
-        const href = parseUrl(hrefVal, options);
-        if (!href || !href.isHTTP) {
-            continue;
-        }
-        const axNode = await page.accessibility.snapshot({ root: $anchor });
-        const textContent = await getProp($anchor, 'textContent', '');
-        const accessibleName = axNode ? axNode.name || '' : textContent.trim();
-        const link = {
-            href,
-            textContent: accessibleName,
-        };
-        anchorList.push(link);
-    }
-    log('Got %d anchors', anchorList.length);
-    dLog('Anchors are: %O', anchorList.map((a) => a.href.href));
-    return anchorList;
-}
-/**
- *
- * @param page
- */
-export async function getMeta(page) {
-    log('Getting Meta');
-    const robotsVal = await getPropBySelector(page, 'meta[name="robots"]', 'content', '');
-    const robots = new Set(robotsVal.split(',').map((robot) => robot.trim().toLowerCase()));
-    const meta = {
-        title: await getPropBySelector(page, 'title', 'textContent', ''),
-        lang: await getPropBySelector(page, 'html', 'lang', ''),
-        description: await getPropBySelector(page, 'meta[name="description"]', 'content', ''),
-        keywords: await getPropBySelector(page, 'meta[name="keywords"]', 'content', ''),
-        noindex: robots.has('noindex'),
-        nofollow: robots.has('nofollow'),
-        noarchive: robots.has('noarchive'),
-        canonical: await getPropBySelector(page, 'link[rel="canonical"]', 'content', ''),
-        alternate: await getPropBySelector(page, 'link[rel="alternate"]', 'content', ''),
-        'og:type': await getPropBySelector(page, 'meta[property="og:type"]', 'content', ''),
-        'og:title': await getPropBySelector(page, 'meta[property="og:title"]', 'content', ''),
-        'og:site_name': await getPropBySelector(page, 'meta[property="og:site_name"]', 'content', ''),
-        'og:description': await getPropBySelector(page, 'meta[property="og:description"]', 'content', ''),
-        'og:url': await getPropBySelector(page, 'meta[property="og:url"]', 'content', ''),
-        'og:image': await getPropBySelector(page, 'meta[property="og:image"]', 'content', ''),
-        'twitter:card': await getPropBySelector(page, 'meta[name="twitter:card"]', 'content', ''),
-    };
-    log('Got meta');
-    dLog('Meta data are: %O', meta);
-    return meta;
-}

package/dist/events.d.ts

@@ -1,32 +0,0 @@
-export declare const subProcessEvent: {
-    start: import("typescript-fsa").ActionCreator<{
-        url: import("./types.js").ExURL;
-        isExternal: boolean;
-        isGettingImages: boolean;
-        excludeKeywords: string[];
-        executablePath: string | null;
-        isSkip: boolean;
-        isTitleOnly: boolean;
-        screenshot: string | null;
-    } & Required<import("./types.js").ParseURLOptions>>;
-    destroy: import("typescript-fsa").ActionCreator<void>;
-};
-export declare const scraperEvent: {
-    ignoreAndSkip: import("typescript-fsa").ActionCreator<import("./types.js").ScrapeEvent & {
-        reason: {
-            matchedText: string;
-            excludeKeywords: string[];
-        };
-    }>;
-    resourceResponse: import("typescript-fsa").ActionCreator<import("./types.js").ScrapeEvent & {
-        log: import("./types.js").NetworkLog;
-        resource: Omit<import("./types.js").Resource, "uid">;
-    }>;
-    scrapeEnd: import("typescript-fsa").ActionCreator<import("./types.js").ScrapeEvent & {
-        timestamp: number;
-        result: import("./types.js").PageData;
-    }>;
-    destroyed: import("typescript-fsa").ActionCreator<Omit<import("./types.js").ScrapeEvent, "url">>;
-    error: import("typescript-fsa").ActionCreator<import("./types.js").ScrapeErrorEvent>;
-    changePhase: import("typescript-fsa").ActionCreator<import("./types.js").ChangePhaseEvent>;
-};

package/dist/events.js

@@ -1,15 +0,0 @@
-import { actionCreatorFactory } from 'typescript-fsa';
-const scraperEventCreator = actionCreatorFactory('@@scraper');
-const subProcessEventCreator = actionCreatorFactory('@@sub-process');
-export const subProcessEvent = {
-    start: subProcessEventCreator('start'),
-    destroy: subProcessEventCreator('destroy'),
-};
-export const scraperEvent = {
-    ignoreAndSkip: scraperEventCreator('ignoreAndSkip'),
-    resourceResponse: scraperEventCreator('resourceResponse'),
-    scrapeEnd: scraperEventCreator('scrapeEnd'),
-    destroyed: scraperEventCreator('destroyed'),
-    error: scraperEventCreator('error'),
-    changePhase: scraperEventCreator('changePhase'),
-};

package/dist/fetch-destination.d.ts

@@ -1,8 +0,0 @@
-import type { PageData, ExURL } from './types.js';
-/**
- *
- * @param url
- * @param isExternal
- * @param method
- */
-export declare function fetchDestination(url: ExURL, isExternal: boolean, method?: string): Promise<PageData>;

package/dist/fetch-destination.js

@@ -1,145 +0,0 @@
-import { delay } from '@d-zero/shared/delay';
-import redirects from 'follow-redirects';
-import NetTimeoutError from './net-timeout-error.js';
-const cacheMap = new Map();
-/**
- *
- * @param url
- * @param isExternal
- * @param method
- */
-export async function fetchDestination(url, isExternal, method = 'HEAD') {
-    if (cacheMap.has(url.withoutHash)) {
-        const cache = cacheMap.get(url.withoutHash);
-        if (cache instanceof Error) {
-            throw cache;
-        }
-        return cache;
-    }
-    const result = await Promise.race([
-        _fetchHead(url, isExternal, method).catch((error) => new Error(error)),
-        (async () => {
-            await delay(10 * 1000);
-            return new NetTimeoutError();
-        })(),
-    ]);
-    cacheMap.set(url.withoutHash, result);
-    if (result instanceof Error) {
-        throw result;
-    }
-    return result;
-}
-/**
- *
- * @param url
- * @param isExternal
- * @param method
- */
-async function _fetchHead(url, isExternal, method) {
-    return new Promise((resolve, reject) => {
-        const hostHeader = url.port ? `${url.hostname}:${url.port}` : url.hostname;
-        const request = {
-            protocol: url.protocol,
-            hostname: url.hostname,
-            port: url.port || undefined,
-            path: url.pathname,
-            method,
-            headers: {
-                host: hostHeader,
-                Connection: 'keep-alive',
-                Pragma: 'no-cache',
-                'Cache-Control': 'no-cache',
-                'Upgrade-Insecure-Requests': 1,
-                // TODO: 'User-Agent': userAgent,
-                Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', // cspell:disable-line
-                'Accept-Encoding': 'gzip, deflate',
-                'Accept-Language': 'ja,en;q=0.9,zh;q=0.8,en-US;q=0.7,pl;q=0.6,de;q=0.5,zh-CN;q=0.4,zh-TW;q=0.3,th;q=0.2,ko;q=0.1,fr;q=0.1',
-            },
-        };
-        if (url.username && url.password) {
-            request.auth = `${url.username}:${url.password}`;
-        }
-        let req;
-        const response = (res) => {
-            res.on('data', () => { });
-            res.on('end', async () => {
-                const redirectPaths = res.redirects.map((r) => r.url);
-                const _contentLength = Number.parseInt(res.headers['content-length'] || '');
-                const contentLength = Number.isFinite(_contentLength) ? _contentLength : null;
-                let rep = {
-                    url,
-                    isTarget: !isExternal,
-                    isExternal,
-                    redirectPaths,
-                    status: res.statusCode || 0,
-                    statusText: res.statusMessage || '',
-                    contentType: res.headers['content-type']?.split(';')[0] || null,
-                    contentLength,
-                    responseHeaders: res.headers,
-                    meta: {
-                        title: '',
-                    },
-                    imageList: [],
-                    anchorList: [],
-                    html: '',
-                    isSkipped: false,
-                };
-                if (rep.status === 405) {
-                    if (method === 'GET') {
-                        reject(`Method Not Allowed: ${url} ${rep.statusText}`);
-                        return;
-                    }
-                    const rr = await fetchDestination(url, isExternal, 'GET').catch((error) => error);
-                    if (rr) {
-                        rep = rr;
-                    }
-                    else {
-                        reject(rr);
-                    }
-                }
-                if (rep.status === 501) {
-                    if (method === 'GET') {
-                        reject(`Method Not Implemented: ${url} ${rep.statusText}`);
-                        return;
-                    }
-                    await delay(5 * 1000);
-                    const rr = await fetchDestination(url, isExternal, 'GET').catch((error) => error);
-                    if (rr) {
-                        rep = rr;
-                    }
-                    else {
-                        reject(rr);
-                    }
-                }
-                if (rep.status === 503) {
-                    if (method === 'GET') {
-                        reject(`Retrying failed: ${url} ${rep.statusText}`);
-                        return;
-                    }
-                    await delay(5 * 1000);
-                    const rr = await fetchDestination(url, isExternal, 'GET').catch((error) => error);
-                    if (rr) {
-                        rep = rr;
-                    }
-                    else {
-                        reject(rr);
-                    }
-                }
-                resolve(rep);
-            });
-        };
-        if (url.protocol === 'https:') {
-            req = redirects.https.request({
-                ...request,
-                rejectUnauthorized: false,
-            }, response);
-        }
-        else {
-            req = redirects.http.request(request, response);
-        }
-        req.on('error', (error) => {
-            reject(error);
-        });
-        req.end();
-    });
-}

package/dist/index.d.ts

@@ -1,4 +0,0 @@
-export { default as SubProcessRunner } from './sub-process-runner.js';
-export { default as default } from './scraper.js';
-export * from './types.js';
-export * from './events.js';

package/dist/index.js

@@ -1,4 +0,0 @@
-export { default as SubProcessRunner } from './sub-process-runner.js';
-export { default as default } from './scraper.js';
-export * from './types.js';
-export * from './events.js';

package/dist/keyword-check.d.ts

@@ -1,6 +0,0 @@
-/**
- *
- * @param html
- * @param excludeKeywords
- */
-export declare function keywordCheck(html: string, excludeKeywords: string[]): string | false;

package/dist/keyword-check.js

@@ -1,15 +0,0 @@
-import { strToRegex } from '@d-zero/shared/str-to-regex';
-/**
- *
- * @param html
- * @param excludeKeywords
- */
-export function keywordCheck(html, excludeKeywords) {
-    for (const keyword of excludeKeywords) {
-        const pattern = strToRegex(keyword);
-        if (pattern.test(html)) {
-            return keyword;
-        }
-    }
-    return false;
-}

package/dist/net-timeout-error.d.ts

@@ -1,3 +0,0 @@
-export default class NetTimeoutError extends Error {
-    name: string;
-}

package/dist/net-timeout-error.js

@@ -1,3 +0,0 @@
-export default class NetTimeoutError extends Error {
-    name = 'NetTimeoutError';
-}

package/dist/scraper.d.ts

@@ -1,15 +0,0 @@
-import type { ScrapeEventTypes, PageData, ParseURLOptions, ExURL } from './types.js';
-import { TypedAwaitEventEmitter } from '@d-zero/shared/typed-await-event-emitter';
-export type ScraperOptions = {
-    isExternal: boolean;
-    isGettingImages: boolean;
-    excludeKeywords: string[];
-    executablePath: string | null;
-    isTitleOnly: boolean;
-    screenshot: string | null;
-} & ParseURLOptions;
-export default class Scraper extends TypedAwaitEventEmitter<ScrapeEventTypes> {
-    #private;
-    destroy(isExternal: boolean): Promise<void>;
-    scrapeStart(url: ExURL, options?: Partial<ScraperOptions>, isSkip?: boolean): Promise<PageData | undefined>;
-}