@nitpicker/crawler 0.6.3 → 0.6.5-alpha.0

package/lib/crawler/network.js

@@ -1,226 +0,0 @@
-import { delay } from '@d-zero/shared/delay';
-import redirects from 'follow-redirects';
-import NetTimeoutError from './net-timeout-error.js';
-/**
- * In-memory cache of HEAD request results keyed by URL (without hash).
- * Stores either the successful {@link PageData} or the {@link Error} to avoid
- * repeated requests to the same destination.
- */
-const cacheMap = new Map();
-/**
- * Clears the in-memory cache of HTTP request results.
- * Should be called between crawl sessions to prevent memory leaks.
- */
-export function clearDestinationCache() {
-    cacheMap.clear();
-}
-/**
- * Fetches the destination metadata for a URL using an HTTP HEAD request (or GET as fallback).
- *
- * Results are cached in memory so that repeated calls for the same URL
- * (without hash) return immediately. The request races against a 10-second
- * timeout; if the server does not respond in time, a {@link NetTimeoutError} is thrown.
- *
- * If the server returns 405 (Method Not Allowed), 501 (Not Implemented), or 503
- * (Service Unavailable) for a HEAD request, the function automatically retries with GET.
- * @param url - The extended URL to fetch.
- * @param isExternal - Whether the URL is external to the crawl scope.
- * @param method - The HTTP method to use. Defaults to `"HEAD"`.
- * @param options - Additional options.
- * @param options.titleBytesLimit - When set, forces a GET request and reads up to this many
- *   bytes from the response body to extract an HTML `<title>` tag. The connection is
- *   destroyed as soon as the limit is reached or a title is found.
- * @returns The page metadata obtained from the HTTP response.
- * @throws {NetTimeoutError} If the request exceeds the 10-second timeout.
- * @throws {Error} If the HTTP request fails for any other reason.
- */
-export async function fetchDestination(url, isExternal, method = 'HEAD', options) {
-    const titleBytesLimit = options?.titleBytesLimit;
-    const cacheKey = titleBytesLimit == null ? url.withoutHash : `${url.withoutHash}:title`;
-    if (cacheMap.has(cacheKey)) {
-        const cache = cacheMap.get(cacheKey);
-        if (cache instanceof Error) {
-            throw cache;
-        }
-        return cache;
-    }
-    const effectiveMethod = titleBytesLimit == null ? method : 'GET';
-    const result = await Promise.race([
-        _fetchHead(url, isExternal, effectiveMethod, titleBytesLimit).catch((error) => (error instanceof Error ? error : new Error(String(error)))),
-        (async () => {
-            await delay(10 * 1000);
-            return new NetTimeoutError(url.href);
-        })(),
-    ]);
-    cacheMap.set(cacheKey, result);
-    if (result instanceof Error) {
-        throw result;
-    }
-    return result;
-}
-/**
- * Performs the actual HTTP request to retrieve page metadata.
- *
- * Handles both HTTP and HTTPS protocols via `follow-redirects`, tracks redirect chains,
- * and falls back to GET on certain status codes (405, 501, 503).
- * @param url - The extended URL to request.
- * @param isExternal - Whether the URL is external to the crawl scope.
- * @param method - The HTTP method (`"HEAD"` or `"GET"`).
- * @param titleBytesLimit - When set, reads up to this many bytes from the response body
- *   to extract a `<title>` tag, then destroys the connection.
- * @returns A promise resolving to {@link PageData} with response metadata.
- */
-async function _fetchHead(url, isExternal, method, titleBytesLimit) {
-    return new Promise((resolve, reject) => {
-        const hostHeader = url.port ? `${url.hostname}:${url.port}` : url.hostname;
-        const request = {
-            protocol: url.protocol,
-            hostname: url.hostname,
-            port: url.port || undefined,
-            path: url.pathname,
-            method,
-            headers: {
-                host: hostHeader,
-                Connection: 'keep-alive',
-                Pragma: 'no-cache',
-                'Cache-Control': 'no-cache',
-                'Upgrade-Insecure-Requests': 1,
-                Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
-                'Accept-Encoding': 'gzip, deflate',
-                'Accept-Language': 'ja,en;q=0.9,zh;q=0.8,en-US;q=0.7,pl;q=0.6,de;q=0.5,zh-CN;q=0.4,zh-TW;q=0.3,th;q=0.2,ko;q=0.1,fr;q=0.1',
-                // Range: url.extname?.toLowerCase() === 'pdf' ? 'bytes=0-0' : undefined,
-            },
-        };
-        if (url.username && url.password) {
-            request.auth = `${url.username}:${url.password}`;
-        }
-        let req;
-        let destroyed = false;
-        const response = (res) => {
-            const chunks = [];
-            let totalBytes = 0;
-            let settled = false;
-            const buildPageData = (title) => {
-                const redirectPaths = res.redirects.map((r) => r.url);
-                const _contentLength = Number.parseInt(res.headers['content-length'] || '');
-                const contentLength = Number.isFinite(_contentLength) ? _contentLength : null;
-                return {
-                    url,
-                    isTarget: !isExternal,
-                    isExternal,
-                    redirectPaths,
-                    status: res.statusCode || 0,
-                    statusText: res.statusMessage || '',
-                    contentType: res.headers['content-type']?.split(';')[0] || null,
-                    contentLength,
-                    responseHeaders: res.headers,
-                    meta: { title },
-                    imageList: [],
-                    anchorList: [],
-                    html: '',
-                    isSkipped: false,
-                };
-            };
-            if (titleBytesLimit == null) {
-                res.on('data', () => { });
-                res.on('end', async () => {
-                    let rep = buildPageData('');
-                    if (rep.status === 405) {
-                        if (method === 'GET') {
-                            reject(new Error(`Method Not Allowed: ${url.href} ${rep.statusText}`));
-                            return;
-                        }
-                        try {
-                            rep = await fetchDestination(url, isExternal, 'GET');
-                        }
-                        catch (error) {
-                            reject(error);
-                            return;
-                        }
-                    }
-                    if (rep.status === 501) {
-                        if (method === 'GET') {
-                            reject(new Error(`Method Not Implemented: ${url.href} ${rep.statusText}`));
-                            return;
-                        }
-                        await delay(5 * 1000);
-                        try {
-                            rep = await fetchDestination(url, isExternal, 'GET');
-                        }
-                        catch (error) {
-                            reject(error);
-                            return;
-                        }
-                    }
-                    if (rep.status === 503) {
-                        if (method === 'GET') {
-                            reject(new Error(`Retrying failed: ${url.href} ${rep.statusText}`));
-                            return;
-                        }
-                        await delay(5 * 1000);
-                        try {
-                            rep = await fetchDestination(url, isExternal, 'GET');
-                        }
-                        catch (error) {
-                            reject(error);
-                            return;
-                        }
-                    }
-                    resolve(rep);
-                });
-            }
-            else {
-                res.on('data', (chunk) => {
-                    if (settled)
-                        return;
-                    chunks.push(chunk);
-                    totalBytes += chunk.length;
-                    // Check for title in accumulated data so far
-                    const body = Buffer.concat(chunks).toString('utf8');
-                    const titleMatch = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(body);
-                    if (titleMatch) {
-                        settled = true;
-                        const title = titleMatch[1]?.trim() ?? '';
-                        resolve(buildPageData(title));
-                        destroyed = true;
-                        req.destroy();
-                        return;
-                    }
-                    // Reached byte limit without finding title
-                    if (totalBytes >= titleBytesLimit) {
-                        settled = true;
-                        resolve(buildPageData(''));
-                        destroyed = true;
-                        req.destroy();
-                    }
-                });
-                res.on('end', () => {
-                    if (settled)
-                        return;
-                    settled = true;
-                    // Stream ended before limit — try to extract title from what we have
-                    const body = Buffer.concat(chunks).toString('utf8');
-                    const titleMatch = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(body);
-                    const title = titleMatch?.[1]?.trim() ?? '';
-                    resolve(buildPageData(title));
-                });
-            }
-        };
-        if (url.protocol === 'https:') {
-            req = redirects.https.request({
-                ...request,
-                rejectUnauthorized: false,
-            }, response);
-        }
-        else {
-            req = redirects.http.request(request, response);
-        }
-        req.on('error', (error) => {
-            // Ignore errors caused by intentional req.destroy()
-            if (destroyed)
-                return;
-            reject(error);
-        });
-        req.end();
-    });
-}

package/lib/crawler/result-handler.d.ts

@@ -1,118 +0,0 @@
-import type LinkList from './link-list.js';
-import type { Link, PageData, Resource } from '../utils/index.js';
-import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
-/**
- * Configuration options that control crawler behavior.
- *
- * Used by the result handler functions to determine how to process
- * scrape results, which URLs to follow, and how to handle external links.
- * @see {@link ./crawler.ts | Crawler} for the main consumer of this type
- * @see {@link ../crawler-orchestrator.ts | CrawlerOrchestrator} for factory methods that build these options
- */
-export type CrawlerOptions = {
-    /** Delay in milliseconds between page requests. */
-    interval: number;
-    /** Maximum number of concurrent scraping processes. 0 uses the default. */
-    parallels: number;
-    /** Whether to recursively follow discovered links within the scope. */
-    recursive: boolean;
-    /** Whether the crawl was started from a pre-defined URL list. */
-    fromList: boolean;
-    /** Whether to capture image resources during scraping. */
-    isGettingImages: boolean;
-    /** Path to the Chromium/Chrome executable, or `null` for the bundled version. */
-    executablePath: string | null;
-    /** Whether to fetch and scrape external (out-of-scope) pages. */
-    fetchExternal: boolean;
-    /** List of scope URL strings that define the crawl boundary. */
-    scope: string[];
-    /** Glob patterns for URLs to exclude from crawling. */
-    excludes: string[];
-    /** Keywords that trigger page exclusion when found in content. */
-    excludeKeywords: string[];
-    /** URL prefixes to exclude from crawling (merged defaults + user additions). */
-    excludeUrls: readonly string[];
-    /** Maximum directory depth for crawling avoidance heuristics. */
-    depthOnAvoid: number;
-    /** Maximum number of retry attempts per URL on scrape failure. */
-    retry: number;
-    /** Whether to enable verbose logging. */
-    verbose: boolean;
-} & Required<Pick<ParseURLOptions, 'disableQueries'>>;
-/**
- * Process the result of a successful page scrape.
- *
- * Extracts anchors from the page (unless in title-only mode), enqueues
- * newly discovered URLs via the `addUrl` callback, and marks the URL
- * as done in the link list.
- * @param result - The scraped page data.
- * @param linkList - The link list managing the crawl queue.
- * @param scope - Map of hostnames to their scope URLs.
- * @param options - Crawler configuration options.
- * @param addUrl - Callback to enqueue a newly discovered URL. Accepts optional
- *   `{ titleOnly: true }` to request metadata-only scraping.
- * @returns An object containing the constructed link and whether the page is external.
- */
-export declare function handleScrapeEnd(result: PageData, linkList: LinkList, scope: ReadonlyMap<string, readonly ExURL[]>, options: CrawlerOptions, addUrl: (url: ExURL, opts?: {
-    titleOnly?: true;
-}) => void): {
-    link: Link | null;
-    isExternal: boolean;
-};
-/**
- * Handle a URL that was ignored or skipped during scraping.
- *
- * Marks the URL as done in the link list without any page data,
- * effectively recording that it was encountered but not scraped.
- * @param url - The URL that was skipped.
- * @param linkList - The link list managing the crawl queue.
- * @param scope - Map of hostnames to their scope URLs.
- * @param options - Crawler configuration options.
- * @returns The constructed {@link Link} object, or `null` if the URL was not in the queue.
- */
-export declare function handleIgnoreAndSkip(url: ExURL, linkList: LinkList, scope: ReadonlyMap<string, readonly ExURL[]>, options: CrawlerOptions): Link | null;
-/**
- * Track a network resource response and determine if it is newly discovered.
- *
- * Checks whether the resource URL has already been seen. If it is new,
- * adds it to the known resources set.
- * @param resource - The captured network resource data.
- * @param resources - The set of already-known resource URLs (without hash).
- * @returns An object with `isNew` indicating whether this resource was seen for the first time.
- */
-export declare function handleResourceResponse(resource: Resource, resources: Set<string>): {
-    isNew: boolean;
-};
-/**
- * Handle an error that occurred during page scraping.
- *
- * Marks the URL as done and creates a fallback {@link PageData} from the
- * link, regardless of whether the error caused a shutdown. This ensures
- * that errored URLs are recorded in the DB (`status = -1, scraped = 1`)
- * and not re-queued on resume.
- * @param payload - The error payload from the scraper.
- * @param payload.url - The URL being scraped when the error occurred, or `null`.
- * @param payload.error - The error details including name, message, and optional stack.
- * @param payload.error.name
- * @param payload.error.message
- * @param payload.error.stack
- * @param payload.shutdown - Whether the error caused the scraper process to shut down.
- * @param payload.pid - The process ID of the scraper, or `undefined`.
- * @param linkList - The link list managing the crawl queue.
- * @param scope - Map of hostnames to their scope URLs.
- * @param options - Crawler configuration options.
- * @returns An object with the link and an optional fallback PageData result.
- */
-export declare function handleScrapeError(payload: {
-    url: ExURL | null;
-    error: {
-        name: string;
-        message: string;
-        stack?: string;
-    };
-    shutdown: boolean;
-    pid: number | undefined;
-}, linkList: LinkList, scope: ReadonlyMap<string, readonly ExURL[]>, options: CrawlerOptions): {
-    link: Link | null;
-    result?: PageData;
-};

package/lib/crawler/result-handler.js

@@ -1,153 +0,0 @@
-import { crawlerErrorLog, crawlerLog } from '../debug.js';
-import { linkToPageData } from './link-to-page-data.js';
-import { injectScopeAuth } from './inject-scope-auth.js';
-import { isExternalUrl } from './is-external-url.js';
-import { isInAnyLowerLayer } from './is-in-any-lower-layer.js';
-/**
- * Process the result of a successful page scrape.
- *
- * Extracts anchors from the page (unless in title-only mode), enqueues
- * newly discovered URLs via the `addUrl` callback, and marks the URL
- * as done in the link list.
- * @param result - The scraped page data.
- * @param linkList - The link list managing the crawl queue.
- * @param scope - Map of hostnames to their scope URLs.
- * @param options - Crawler configuration options.
- * @param addUrl - Callback to enqueue a newly discovered URL. Accepts optional
- *   `{ titleOnly: true }` to request metadata-only scraping.
- * @returns An object containing the constructed link and whether the page is external.
- */
-export function handleScrapeEnd(result, linkList, scope, options, addUrl) {
-    const isTitleOnly = linkList.isTitleOnly(result.url.withoutHash);
-    if (!isTitleOnly) {
-        processAnchors(result.anchorList, scope, options, addUrl);
-    }
-    const link = linkList.done(result.url, scope, {
-        page: result,
-    }, options);
-    crawlerLog('Scrape end URL: %s', result.url.href);
-    crawlerLog('Scrape end Status: %d', result.status);
-    crawlerLog('Scrape end Type: %s', result.contentType);
-    if (!result.isExternal) {
-        crawlerLog('Scrape end Anchors: %d URLs', result.anchorList.length);
-    }
-    return { link, isExternal: result.isExternal };
-}
-/**
- * Handle a URL that was ignored or skipped during scraping.
- *
- * Marks the URL as done in the link list without any page data,
- * effectively recording that it was encountered but not scraped.
- * @param url - The URL that was skipped.
- * @param linkList - The link list managing the crawl queue.
- * @param scope - Map of hostnames to their scope URLs.
- * @param options - Crawler configuration options.
- * @returns The constructed {@link Link} object, or `null` if the URL was not in the queue.
- */
-export function handleIgnoreAndSkip(url, linkList, scope, options) {
-    const updated = linkList.done(url, scope, {}, options);
-    if (updated) {
-        crawlerLog('Skipped URL: %s', url.href);
-    }
-    return updated;
-}
-/**
- * Track a network resource response and determine if it is newly discovered.
- *
- * Checks whether the resource URL has already been seen. If it is new,
- * adds it to the known resources set.
- * @param resource - The captured network resource data.
- * @param resources - The set of already-known resource URLs (without hash).
- * @returns An object with `isNew` indicating whether this resource was seen for the first time.
- */
-export function handleResourceResponse(resource, resources) {
-    const isNew = !resources.has(resource.url.withoutHash);
-    if (isNew) {
-        resources.add(resource.url.withoutHash);
-    }
-    return { isNew };
-}
-/**
- * Handle an error that occurred during page scraping.
- *
- * Marks the URL as done and creates a fallback {@link PageData} from the
- * link, regardless of whether the error caused a shutdown. This ensures
- * that errored URLs are recorded in the DB (`status = -1, scraped = 1`)
- * and not re-queued on resume.
- * @param payload - The error payload from the scraper.
- * @param payload.url - The URL being scraped when the error occurred, or `null`.
- * @param payload.error - The error details including name, message, and optional stack.
- * @param payload.error.name
- * @param payload.error.message
- * @param payload.error.stack
- * @param payload.shutdown - Whether the error caused the scraper process to shut down.
- * @param payload.pid - The process ID of the scraper, or `undefined`.
- * @param linkList - The link list managing the crawl queue.
- * @param scope - Map of hostnames to their scope URLs.
- * @param options - Crawler configuration options.
- * @returns An object with the link and an optional fallback PageData result.
- */
-export function handleScrapeError(payload, linkList, scope, options) {
-    const { url, error, shutdown, pid } = payload;
-    let link = null;
-    let result;
-    if (url) {
-        const updated = linkList.done(url, scope, { error }, options);
-        if (updated) {
-            link = updated;
-            result = linkToPageData(updated);
-        }
-    }
-    crawlerErrorLog('From %d(%s)', pid, url?.href ?? 'UNKNOWN_URL');
-    crawlerErrorLog('Then shutdown?: %s', shutdown ? 'Yes' : 'No');
-    crawlerErrorLog('%O', error);
-    return { link, result };
-}
-/**
- * Process anchor elements extracted from a scraped page and enqueue new URLs.
- *
- * For each anchor:
- * 1. Determines if it is external (outside the crawl scope)
- * 2. Injects authentication credentials from matching scope URLs
- * 3. Reconstructs the `withoutHash` URL with injected auth
- * 4. In recursive mode: enqueues internal lower-layer URLs for full scraping,
- *    and external URLs for title-only scraping (if `fetchExternal` is enabled)
- * 5. In non-recursive mode: enqueues all URLs for title-only scraping
- * @param anchors - The list of anchor data extracted from the page.
- * @param scope - Map of hostnames to their scope URLs.
- * @param options - Crawler configuration options.
- * @param addUrl - Callback to enqueue a newly discovered URL. Accepts optional
- *   `{ titleOnly: true }` to request metadata-only scraping.
- */
-function processAnchors(anchors, scope, options, addUrl) {
-    for (const anchor of anchors) {
-        const isExternal = isExternalUrl(anchor.href, scope);
-        anchor.isExternal = isExternal;
-        if (!isExternal && (!anchor.href.username || !anchor.href.password)) {
-            injectScopeAuth(anchor.href, scope);
-            const auth = anchor.href.username && anchor.href.password
-                ? `${anchor.href.username}:${anchor.href.password}@`
-                : '';
-            const host = anchor.href.hostname + (anchor.href.port ? `:${anchor.href.port}` : '');
-            const newSearch = anchor.href.query ? `?${anchor.href.query}` : '';
-            const body = anchor.href.dirname
-                ? `${anchor.href.paths.join('/')}${newSearch}`
-                : newSearch
-                    ? `${newSearch}`
-                    : '';
-            const withoutHash = `${anchor.href.protocol}//${auth}${host}${body ? `/${body}` : ''}`;
-            anchor.href.withoutHash = withoutHash;
-        }
-        if (options.recursive) {
-            const scopes = scope.get(anchor.href.hostname);
-            if (scopes && isInAnyLowerLayer(anchor.href, scopes, options)) {
-                addUrl(anchor.href);
-            }
-            else if (isExternal && options.fetchExternal) {
-                addUrl(anchor.href, { titleOnly: true });
-            }
-            continue;
-        }
-        addUrl(anchor.href, { titleOnly: true });
-    }
-}

package/lib/crawler/speculative-pagination.d.ts

@@ -1,52 +0,0 @@
-import type { ScrapeResult } from '@nitpicker/beholder';
-/**
- * Describes a detected pagination pattern between two consecutive URLs.
- */
-export interface PaginationPattern {
-    /** Index within the combined token array (path segments + query values) where the numeric difference was found. */
-    tokenIndex: number;
-    /** The numeric increment (always > 0). */
-    step: number;
-    /** The number found at `tokenIndex` in the "current" URL. */
-    currentNumber: number;
-}
-/**
- * Compares two consecutive URL strings and detects a single-token numeric
- * pagination pattern (e.g. `/page/1` → `/page/2`, or `?p=1` → `?p=2`).
- *
- * The algorithm decomposes each URL into tokens (path segments + sorted query values),
- * then checks that exactly one token differs and both values are integers with a
- * positive step. Returns `null` when no pattern is detected.
- *
- * WHY single-token constraint: Multi-token differences (e.g. both path and query
- * changing) indicate different routes rather than pagination, so they are rejected.
- * @param prevUrl - The previously pushed URL (protocol-agnostic, without hash/auth)
- * @param currentUrl - The newly discovered URL
- * @returns The detected pattern, or `null` if no pagination pattern was found
- */
-export declare function detectPaginationPattern(prevUrl: string, currentUrl: string): PaginationPattern | null;
-/**
- * Generates speculative URLs by extrapolating the detected pagination pattern.
- *
- * Starting from `currentUrl`, applies the pattern's step `count` times to produce
- * future page URLs (e.g. if step=1 and currentNumber=2, generates page 3, 4, ...).
- * These URLs are pushed into the crawl queue and discarded later if they 404.
- * @param pattern - The detected pagination pattern from {@link detectPaginationPattern}
- * @param currentUrl - The URL to extrapolate from (protocol-agnostic, without hash/auth)
- * @param count - Number of speculative URLs to generate (typically equals concurrency)
- * @returns Array of speculative URL strings
- */
-export declare function generateSpeculativeUrls(pattern: PaginationPattern, currentUrl: string, count: number): string[];
-/**
- * Determines whether a speculative URL's scrape result should be discarded.
- *
- * Speculative URLs are pre-emptively pushed into the crawl queue before
- * knowing if they exist. This function filters out invalid results:
- * - `error` type → discard (server unreachable, timeout, etc.)
- * - `ignoreAndSkip` type → discard (matched exclusion rule)
- * - `scrapeEnd` with HTTP error status (4xx/5xx) → discard
- * - `scrapeEnd` with 2xx/3xx → keep
- * @param result - The scrape result for the speculative URL
- * @returns `true` if the result should be discarded (not saved to archive)
- */
-export declare function shouldDiscardSpeculative(result: ScrapeResult): boolean;