@nitpicker/crawler 0.4.1

package/lib/crawler/result-handler.d.ts

@@ -0,0 +1,118 @@
+import type LinkList from './link-list.js';
+import type { Link, PageData, Resource } from '../utils/index.js';
+import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
+/**
+ * Configuration options that control crawler behavior.
+ *
+ * Used by the result handler functions to determine how to process
+ * scrape results, which URLs to follow, and how to handle external links.
+ * @see {@link ./crawler.ts | Crawler} for the main consumer of this type
+ * @see {@link ../crawler-orchestrator.ts | CrawlerOrchestrator} for factory methods that build these options
+ */
+export type CrawlerOptions = {
+    /** Delay in milliseconds between page requests. */
+    interval: number;
+    /** Maximum number of concurrent scraping processes. 0 uses the default. */
+    parallels: number;
+    /** Whether to recursively follow discovered links within the scope. */
+    recursive: boolean;
+    /** Whether the crawl was started from a pre-defined URL list. */
+    fromList: boolean;
+    /** Whether to capture image resources during scraping. */
+    isGettingImages: boolean;
+    /** Path to the Chromium/Chrome executable, or `null` for the bundled version. */
+    executablePath: string | null;
+    /** Whether to fetch and scrape external (out-of-scope) pages. */
+    fetchExternal: boolean;
+    /** List of scope URL strings that define the crawl boundary. */
+    scope: string[];
+    /** Glob patterns for URLs to exclude from crawling. */
+    excludes: string[];
+    /** Keywords that trigger page exclusion when found in content. */
+    excludeKeywords: string[];
+    /** URL prefixes to exclude from crawling (merged defaults + user additions). */
+    excludeUrls: readonly string[];
+    /** Maximum directory depth for crawling avoidance heuristics. */
+    depthOnAvoid: number;
+    /** Maximum number of retry attempts per URL on scrape failure. */
+    retry: number;
+    /** Whether to enable verbose logging. */
+    verbose: boolean;
+} & Required<Pick<ParseURLOptions, 'disableQueries'>>;
+/**
+ * Process the result of a successful page scrape.
+ *
+ * Extracts anchors from the page (unless in title-only mode), enqueues
+ * newly discovered URLs via the `addUrl` callback, and marks the URL
+ * as done in the link list.
+ * @param result - The scraped page data.
+ * @param linkList - The link list managing the crawl queue.
+ * @param scope - Map of hostnames to their scope URLs.
+ * @param options - Crawler configuration options.
+ * @param addUrl - Callback to enqueue a newly discovered URL. Accepts optional
+ *   `{ titleOnly: true }` to request metadata-only scraping.
+ * @returns An object containing the constructed link and whether the page is external.
+ */
+export declare function handleScrapeEnd(result: PageData, linkList: LinkList, scope: ReadonlyMap<string, readonly ExURL[]>, options: CrawlerOptions, addUrl: (url: ExURL, opts?: {
+    titleOnly?: true;
+}) => void): {
+    link: Link | null;
+    isExternal: boolean;
+};
+/**
+ * Handle a URL that was ignored or skipped during scraping.
+ *
+ * Marks the URL as done in the link list without any page data,
+ * effectively recording that it was encountered but not scraped.
+ * @param url - The URL that was skipped.
+ * @param linkList - The link list managing the crawl queue.
+ * @param scope - Map of hostnames to their scope URLs.
+ * @param options - Crawler configuration options.
+ * @returns The constructed {@link Link} object, or `null` if the URL was not in the queue.
+ */
+export declare function handleIgnoreAndSkip(url: ExURL, linkList: LinkList, scope: ReadonlyMap<string, readonly ExURL[]>, options: CrawlerOptions): Link | null;
+/**
+ * Track a network resource response and determine if it is newly discovered.
+ *
+ * Checks whether the resource URL has already been seen. If it is new,
+ * adds it to the known resources set.
+ * @param resource - The captured network resource data.
+ * @param resources - The set of already-known resource URLs (without hash).
+ * @returns An object with `isNew` indicating whether this resource was seen for the first time.
+ */
+export declare function handleResourceResponse(resource: Resource, resources: Set<string>): {
+    isNew: boolean;
+};
+/**
+ * Handle an error that occurred during page scraping.
+ *
+ * Marks the URL as done and creates a fallback {@link PageData} from the
+ * link, regardless of whether the error caused a shutdown. This ensures
+ * that errored URLs are recorded in the DB (`status = -1, scraped = 1`)
+ * and not re-queued on resume.
+ * @param payload - The error payload from the scraper.
+ * @param payload.url - The URL being scraped when the error occurred, or `null`.
+ * @param payload.error - The error details including name, message, and optional stack.
+ * @param payload.error.name
+ * @param payload.error.message
+ * @param payload.error.stack
+ * @param payload.shutdown - Whether the error caused the scraper process to shut down.
+ * @param payload.pid - The process ID of the scraper, or `undefined`.
+ * @param linkList - The link list managing the crawl queue.
+ * @param scope - Map of hostnames to their scope URLs.
+ * @param options - Crawler configuration options.
+ * @returns An object with the link and an optional fallback PageData result.
+ */
+export declare function handleScrapeError(payload: {
+    url: ExURL | null;
+    error: {
+        name: string;
+        message: string;
+        stack?: string;
+    };
+    shutdown: boolean;
+    pid: number | undefined;
+}, linkList: LinkList, scope: ReadonlyMap<string, readonly ExURL[]>, options: CrawlerOptions): {
+    link: Link | null;
+    result?: PageData;
+};

package/lib/crawler/result-handler.js

@@ -0,0 +1,153 @@
+import { crawlerErrorLog, crawlerLog } from '../debug.js';
+import { linkToPageData } from './link-to-page-data.js';
+import { injectScopeAuth } from './inject-scope-auth.js';
+import { isExternalUrl } from './is-external-url.js';
+import { isInAnyLowerLayer } from './is-in-any-lower-layer.js';
+/**
+ * Process the result of a successful page scrape.
+ *
+ * Extracts anchors from the page (unless in title-only mode), enqueues
+ * newly discovered URLs via the `addUrl` callback, and marks the URL
+ * as done in the link list.
+ * @param result - The scraped page data.
+ * @param linkList - The link list managing the crawl queue.
+ * @param scope - Map of hostnames to their scope URLs.
+ * @param options - Crawler configuration options.
+ * @param addUrl - Callback to enqueue a newly discovered URL. Accepts optional
+ *   `{ titleOnly: true }` to request metadata-only scraping.
+ * @returns An object containing the constructed link and whether the page is external.
+ */
+export function handleScrapeEnd(result, linkList, scope, options, addUrl) {
+    const isTitleOnly = linkList.isTitleOnly(result.url.withoutHash);
+    if (!isTitleOnly) {
+        processAnchors(result.anchorList, scope, options, addUrl);
+    }
+    const link = linkList.done(result.url, scope, {
+        page: result,
+    }, options);
+    crawlerLog('Scrape end URL: %s', result.url.href);
+    crawlerLog('Scrape end Status: %d', result.status);
+    crawlerLog('Scrape end Type: %s', result.contentType);
+    if (!result.isExternal) {
+        crawlerLog('Scrape end Anchors: %d URLs', result.anchorList.length);
+    }
+    return { link, isExternal: result.isExternal };
+}
+/**
+ * Handle a URL that was ignored or skipped during scraping.
+ *
+ * Marks the URL as done in the link list without any page data,
+ * effectively recording that it was encountered but not scraped.
+ * @param url - The URL that was skipped.
+ * @param linkList - The link list managing the crawl queue.
+ * @param scope - Map of hostnames to their scope URLs.
+ * @param options - Crawler configuration options.
+ * @returns The constructed {@link Link} object, or `null` if the URL was not in the queue.
+ */
+export function handleIgnoreAndSkip(url, linkList, scope, options) {
+    const updated = linkList.done(url, scope, {}, options);
+    if (updated) {
+        crawlerLog('Skipped URL: %s', url.href);
+    }
+    return updated;
+}
+/**
+ * Track a network resource response and determine if it is newly discovered.
+ *
+ * Checks whether the resource URL has already been seen. If it is new,
+ * adds it to the known resources set.
+ * @param resource - The captured network resource data.
+ * @param resources - The set of already-known resource URLs (without hash).
+ * @returns An object with `isNew` indicating whether this resource was seen for the first time.
+ */
+export function handleResourceResponse(resource, resources) {
+    const isNew = !resources.has(resource.url.withoutHash);
+    if (isNew) {
+        resources.add(resource.url.withoutHash);
+    }
+    return { isNew };
+}
+/**
+ * Handle an error that occurred during page scraping.
+ *
+ * Marks the URL as done and creates a fallback {@link PageData} from the
+ * link, regardless of whether the error caused a shutdown. This ensures
+ * that errored URLs are recorded in the DB (`status = -1, scraped = 1`)
+ * and not re-queued on resume.
+ * @param payload - The error payload from the scraper.
+ * @param payload.url - The URL being scraped when the error occurred, or `null`.
+ * @param payload.error - The error details including name, message, and optional stack.
+ * @param payload.error.name
+ * @param payload.error.message
+ * @param payload.error.stack
+ * @param payload.shutdown - Whether the error caused the scraper process to shut down.
+ * @param payload.pid - The process ID of the scraper, or `undefined`.
+ * @param linkList - The link list managing the crawl queue.
+ * @param scope - Map of hostnames to their scope URLs.
+ * @param options - Crawler configuration options.
+ * @returns An object with the link and an optional fallback PageData result.
+ */
+export function handleScrapeError(payload, linkList, scope, options) {
+    const { url, error, shutdown, pid } = payload;
+    let link = null;
+    let result;
+    if (url) {
+        const updated = linkList.done(url, scope, { error }, options);
+        if (updated) {
+            link = updated;
+            result = linkToPageData(updated);
+        }
+    }
+    crawlerErrorLog('From %d(%s)', pid, url?.href ?? 'UNKNOWN_URL');
+    crawlerErrorLog('Then shutdown?: %s', shutdown ? 'Yes' : 'No');
+    crawlerErrorLog('%O', error);
+    return { link, result };
+}
+/**
+ * Process anchor elements extracted from a scraped page and enqueue new URLs.
+ *
+ * For each anchor:
+ * 1. Determines if it is external (outside the crawl scope)
+ * 2. Injects authentication credentials from matching scope URLs
+ * 3. Reconstructs the `withoutHash` URL with injected auth
+ * 4. In recursive mode: enqueues internal lower-layer URLs for full scraping,
+ *    and external URLs for title-only scraping (if `fetchExternal` is enabled)
+ * 5. In non-recursive mode: enqueues all URLs for title-only scraping
+ * @param anchors - The list of anchor data extracted from the page.
+ * @param scope - Map of hostnames to their scope URLs.
+ * @param options - Crawler configuration options.
+ * @param addUrl - Callback to enqueue a newly discovered URL. Accepts optional
+ *   `{ titleOnly: true }` to request metadata-only scraping.
+ */
+function processAnchors(anchors, scope, options, addUrl) {
+    for (const anchor of anchors) {
+        const isExternal = isExternalUrl(anchor.href, scope);
+        anchor.isExternal = isExternal;
+        if (!isExternal && (!anchor.href.username || !anchor.href.password)) {
+            injectScopeAuth(anchor.href, scope);
+            const auth = anchor.href.username && anchor.href.password
+                ? `${anchor.href.username}:${anchor.href.password}@`
+                : '';
+            const host = anchor.href.hostname + (anchor.href.port ? `:${anchor.href.port}` : '');
+            const newSearch = anchor.href.query ? `?${anchor.href.query}` : '';
+            const body = anchor.href.dirname
+                ? `${anchor.href.paths.join('/')}${newSearch}`
+                : newSearch
+                    ? `${newSearch}`
+                    : '';
+            const withoutHash = `${anchor.href.protocol}//${auth}${host}${body ? `/${body}` : ''}`;
+            anchor.href.withoutHash = withoutHash;
+        }
+        if (options.recursive) {
+            const scopes = scope.get(anchor.href.hostname);
+            if (scopes && isInAnyLowerLayer(anchor.href, scopes, options)) {
+                addUrl(anchor.href);
+            }
+            else if (isExternal && options.fetchExternal) {
+                addUrl(anchor.href, { titleOnly: true });
+            }
+            continue;
+        }
+        addUrl(anchor.href, { titleOnly: true });
+    }
+}

package/lib/crawler/robots-checker.d.ts

@@ -0,0 +1,26 @@
+import type { ExURL } from '@d-zero/shared/parse-url';
+/**
+ * Checks whether a URL is allowed by the site's robots.txt rules.
+ *
+ * Caches robots.txt per origin so each origin is fetched at most once.
+ * When disabled (i.e., `ignoreRobots` mode), all URLs are allowed.
+ */
+export declare class RobotsChecker {
+    #private;
+    /**
+     * Create a new RobotsChecker.
+     * @param userAgent - User-Agent string for rule matching and fetching robots.txt.
+     * @param enabled - Whether robots.txt checking is enabled. When `false`, {@link isAllowed} always returns `true`.
+     */
+    constructor(userAgent: string, enabled: boolean);
+    /**
+     * Check whether the given URL is allowed by the site's robots.txt.
+     *
+     * Fetches and caches robots.txt per origin on first access.
+     * Returns `true` if robots.txt checking is disabled, if no robots.txt
+     * exists, or if the URL is explicitly allowed.
+     * @param url - The URL to check.
+     * @returns `true` if the URL is allowed, `false` if blocked.
+     */
+    isAllowed(url: ExURL): Promise<boolean>;
+}

package/lib/crawler/robots-checker.js

@@ -0,0 +1,62 @@
+import { crawlerLog } from '../debug.js';
+import { fetchRobotsTxt } from './fetch-robots-txt.js';
+/**
+ * Derives the origin string from an ExURL (e.g., `https://example.com:8080`).
+ * @param url - The extended URL.
+ * @returns The origin string.
+ */
+function getOrigin(url) {
+    return `${url.protocol}//${url.hostname}${url.port ? `:${url.port}` : ''}`;
+}
+/**
+ * Checks whether a URL is allowed by the site's robots.txt rules.
+ *
+ * Caches robots.txt per origin so each origin is fetched at most once.
+ * When disabled (i.e., `ignoreRobots` mode), all URLs are allowed.
+ */
+export class RobotsChecker {
+    /** Cache of parsed robots.txt per origin. `null` means no robots.txt or fetch failed. */
+    #cache = new Map();
+    /** When `false`, robots.txt checking is disabled and all URLs are allowed. */
+    #enabled;
+    /** User-Agent string used for robots.txt rule matching and HTTP requests. */
+    #userAgent;
+    /**
+     * Create a new RobotsChecker.
+     * @param userAgent - User-Agent string for rule matching and fetching robots.txt.
+     * @param enabled - Whether robots.txt checking is enabled. When `false`, {@link isAllowed} always returns `true`.
+     */
+    constructor(userAgent, enabled) {
+        this.#userAgent = userAgent;
+        this.#enabled = enabled;
+    }
+    /**
+     * Check whether the given URL is allowed by the site's robots.txt.
+     *
+     * Fetches and caches robots.txt per origin on first access.
+     * Returns `true` if robots.txt checking is disabled, if no robots.txt
+     * exists, or if the URL is explicitly allowed.
+     * @param url - The URL to check.
+     * @returns `true` if the URL is allowed, `false` if blocked.
+     */
+    async isAllowed(url) {
+        if (!this.#enabled) {
+            return true;
+        }
+        if (!url.isHTTP) {
+            return true;
+        }
+        const origin = getOrigin(url);
+        if (!this.#cache.has(origin)) {
+            crawlerLog('Fetching robots.txt for %s', origin);
+            const robot = await fetchRobotsTxt(origin, this.#userAgent);
+            this.#cache.set(origin, robot);
+        }
+        const robot = this.#cache.get(origin);
+        if (!robot) {
+            return true;
+        }
+        const allowed = robot.isAllowed(url.href, this.#userAgent);
+        return allowed !== false;
+    }
+}

package/lib/crawler/should-discard-predicted.d.ts

@@ -0,0 +1,14 @@
+import type { ScrapeResult } from '@d-zero/beholder';
+/**
+ * Determines whether a predicted URL's scrape result should be discarded.
+ *
+ * Predicted URLs are pre-emptively pushed into the crawl queue before
+ * knowing if they exist. This function filters out invalid results:
+ * - `error` type → discard (server unreachable, timeout, etc.)
+ * - `skipped` type → discard (matched exclusion rule)
+ * - `success` with HTTP error status (4xx/5xx) → discard
+ * - `success` with 2xx/3xx → keep
+ * @param result - The scrape result for the predicted URL
+ * @returns `true` if the result should be discarded (not saved to archive)
+ */
+export declare function shouldDiscardPredicted(result: ScrapeResult): boolean;

package/lib/crawler/should-discard-predicted.js

@@ -0,0 +1,31 @@
+import { isError } from '@d-zero/beholder';
+/**
+ * Determines whether a predicted URL's scrape result should be discarded.
+ *
+ * Predicted URLs are pre-emptively pushed into the crawl queue before
+ * knowing if they exist. This function filters out invalid results:
+ * - `error` type → discard (server unreachable, timeout, etc.)
+ * - `skipped` type → discard (matched exclusion rule)
+ * - `success` with HTTP error status (4xx/5xx) → discard
+ * - `success` with 2xx/3xx → keep
+ * @param result - The scrape result for the predicted URL
+ * @returns `true` if the result should be discarded (not saved to archive)
+ */
+export function shouldDiscardPredicted(result) {
+    switch (result.type) {
+        case 'error': {
+            return true;
+        }
+        case 'skipped': {
+            return true;
+        }
+        case 'success': {
+            if (!result.pageData)
+                return true;
+            return isError(result.pageData.status);
+        }
+        default: {
+            return true;
+        }
+    }
+}

package/lib/crawler/should-skip-url.d.ts

@@ -0,0 +1,23 @@
+import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
+/**
+ * Parameters for {@link shouldSkipUrl}.
+ */
+export interface ShouldSkipUrlParams {
+    /** The parsed URL to check. */
+    readonly url: ExURL;
+    /** Array of glob patterns for URLs to exclude. */
+    readonly excludes: readonly string[];
+    /** Array of URL prefixes to exclude (matched via `startsWith`). */
+    readonly excludeUrls: readonly string[];
+    /** URL parsing options used for pattern matching. */
+    readonly options: ParseURLOptions;
+}
+/**
+ * Determine whether a URL should be skipped during crawling.
+ *
+ * A URL is skipped if it matches any user-defined exclude glob pattern
+ * or starts with any of the excluded URL prefixes.
+ * @param params - Parameters containing the URL, exclude patterns, and options.
+ * @returns `true` if the URL should be skipped.
+ */
+export declare function shouldSkipUrl(params: ShouldSkipUrlParams): boolean;

package/lib/crawler/should-skip-url.js

@@ -0,0 +1,15 @@
+import { pathMatch } from '@d-zero/shared/path-match';
+import { protocolAgnosticKey } from './protocol-agnostic-key.js';
+/**
+ * Determine whether a URL should be skipped during crawling.
+ *
+ * A URL is skipped if it matches any user-defined exclude glob pattern
+ * or starts with any of the excluded URL prefixes.
+ * @param params - Parameters containing the URL, exclude patterns, and options.
+ * @returns `true` if the URL should be skipped.
+ */
+export function shouldSkipUrl(params) {
+    const { url, excludes, excludeUrls, options } = params;
+    return (excludes.some((excludeGlobPattern) => pathMatch(url, excludeGlobPattern, options)) ||
+        excludeUrls.some((prefix) => protocolAgnosticKey(url.href).startsWith(protocolAgnosticKey(prefix))));
+}

package/lib/crawler/speculative-pagination.d.ts

@@ -0,0 +1,52 @@
+import type { ScrapeResult } from '@nitpicker/beholder';
+/**
+ * Describes a detected pagination pattern between two consecutive URLs.
+ */
+export interface PaginationPattern {
+    /** Index within the combined token array (path segments + query values) where the numeric difference was found. */
+    tokenIndex: number;
+    /** The numeric increment (always > 0). */
+    step: number;
+    /** The number found at `tokenIndex` in the "current" URL. */
+    currentNumber: number;
+}
+/**
+ * Compares two consecutive URL strings and detects a single-token numeric
+ * pagination pattern (e.g. `/page/1` → `/page/2`, or `?p=1` → `?p=2`).
+ *
+ * The algorithm decomposes each URL into tokens (path segments + sorted query values),
+ * then checks that exactly one token differs and both values are integers with a
+ * positive step. Returns `null` when no pattern is detected.
+ *
+ * WHY single-token constraint: Multi-token differences (e.g. both path and query
+ * changing) indicate different routes rather than pagination, so they are rejected.
+ * @param prevUrl - The previously pushed URL (protocol-agnostic, without hash/auth)
+ * @param currentUrl - The newly discovered URL
+ * @returns The detected pattern, or `null` if no pagination pattern was found
+ */
+export declare function detectPaginationPattern(prevUrl: string, currentUrl: string): PaginationPattern | null;
+/**
+ * Generates speculative URLs by extrapolating the detected pagination pattern.
+ *
+ * Starting from `currentUrl`, applies the pattern's step `count` times to produce
+ * future page URLs (e.g. if step=1 and currentNumber=2, generates page 3, 4, ...).
+ * These URLs are pushed into the crawl queue and discarded later if they 404.
+ * @param pattern - The detected pagination pattern from {@link detectPaginationPattern}
+ * @param currentUrl - The URL to extrapolate from (protocol-agnostic, without hash/auth)
+ * @param count - Number of speculative URLs to generate (typically equals concurrency)
+ * @returns Array of speculative URL strings
+ */
+export declare function generateSpeculativeUrls(pattern: PaginationPattern, currentUrl: string, count: number): string[];
+/**
+ * Determines whether a speculative URL's scrape result should be discarded.
+ *
+ * Speculative URLs are pre-emptively pushed into the crawl queue before
+ * knowing if they exist. This function filters out invalid results:
+ * - `error` type → discard (server unreachable, timeout, etc.)
+ * - `ignoreAndSkip` type → discard (matched exclusion rule)
+ * - `scrapeEnd` with HTTP error status (4xx/5xx) → discard
+ * - `scrapeEnd` with 2xx/3xx → keep
+ * @param result - The scrape result for the speculative URL
+ * @returns `true` if the result should be discarded (not saved to archive)
+ */
+export declare function shouldDiscardSpeculative(result: ScrapeResult): boolean;