@nitpicker/crawler 0.4.1

package/lib/crawler/handle-resource-response.d.ts

@@ -0,0 +1,13 @@
+import type { Resource } from '../utils/index.js';
+/**
+ * Track a network resource response and determine if it is newly discovered.
+ *
+ * Checks whether the resource URL has already been seen. If it is new,
+ * adds it to the known resources set.
+ * @param resource - The captured network resource data.
+ * @param resources - The set of already-known resource URLs (without hash).
+ * @returns An object with `isNew` indicating whether this resource was seen for the first time.
+ */
+export declare function handleResourceResponse(resource: Resource, resources: Set<string>): {
+    isNew: boolean;
+};

package/lib/crawler/handle-resource-response.js

@@ -0,0 +1,16 @@
+/**
+ * Track a network resource response and determine if it is newly discovered.
+ *
+ * Checks whether the resource URL has already been seen. If it is new,
+ * adds it to the known resources set.
+ * @param resource - The captured network resource data.
+ * @param resources - The set of already-known resource URLs (without hash).
+ * @returns An object with `isNew` indicating whether this resource was seen for the first time.
+ */
+export function handleResourceResponse(resource, resources) {
+    const isNew = !resources.has(resource.url.withoutHash);
+    if (isNew) {
+        resources.add(resource.url.withoutHash);
+    }
+    return { isNew };
+}

package/lib/crawler/handle-scrape-end.d.ts

@@ -0,0 +1,24 @@
+import type LinkList from './link-list.js';
+import type { CrawlerOptions } from './types.js';
+import type { Link, PageData } from '../utils/index.js';
+import type { ExURL } from '@d-zero/shared/parse-url';
+/**
+ * Process the result of a successful page scrape.
+ *
+ * Extracts anchors from the page (unless in metadata-only mode), enqueues
+ * newly discovered URLs via the `addUrl` callback, and marks the URL
+ * as done in the link list.
+ * @param result - The scraped page data.
+ * @param linkList - The link list managing the crawl queue.
+ * @param scope - Map of hostnames to their scope URLs.
+ * @param options - Crawler configuration options.
+ * @param addUrl - Callback to enqueue a newly discovered URL. Accepts optional
+ *   `{ metadataOnly: true }` to request metadata-only scraping.
+ * @returns An object containing the constructed link and whether the page is external.
+ */
+export declare function handleScrapeEnd(result: PageData, linkList: LinkList, scope: ReadonlyMap<string, readonly ExURL[]>, options: CrawlerOptions, addUrl: (url: ExURL, opts?: {
+    metadataOnly?: true;
+}) => void): {
+    link: Link | null;
+    isExternal: boolean;
+};

package/lib/crawler/handle-scrape-end.js

@@ -0,0 +1,82 @@
+import { crawlerLog } from '../debug.js';
+import { injectScopeAuth } from './inject-scope-auth.js';
+import { isExternalUrl } from './is-external-url.js';
+import { isInAnyLowerLayer } from './is-in-any-lower-layer.js';
+/**
+ * Process the result of a successful page scrape.
+ *
+ * Extracts anchors from the page (unless in metadata-only mode), enqueues
+ * newly discovered URLs via the `addUrl` callback, and marks the URL
+ * as done in the link list.
+ * @param result - The scraped page data.
+ * @param linkList - The link list managing the crawl queue.
+ * @param scope - Map of hostnames to their scope URLs.
+ * @param options - Crawler configuration options.
+ * @param addUrl - Callback to enqueue a newly discovered URL. Accepts optional
+ *   `{ metadataOnly: true }` to request metadata-only scraping.
+ * @returns An object containing the constructed link and whether the page is external.
+ */
+export function handleScrapeEnd(result, linkList, scope, options, addUrl) {
+    const isMetadataOnly = linkList.isMetadataOnly(result.url.withoutHash);
+    if (!isMetadataOnly) {
+        processAnchors(result.anchorList, scope, options, addUrl);
+    }
+    const link = linkList.done(result.url, scope, {
+        page: result,
+    }, options);
+    crawlerLog('Scrape end URL: %s', result.url.href);
+    crawlerLog('Scrape end Status: %d', result.status);
+    crawlerLog('Scrape end Type: %s', result.contentType);
+    if (!result.isExternal) {
+        crawlerLog('Scrape end Anchors: %d URLs', result.anchorList.length);
+    }
+    return { link, isExternal: result.isExternal };
+}
+/**
+ * Process anchor elements extracted from a scraped page and enqueue new URLs.
+ *
+ * For each anchor:
+ * 1. Determines if it is external (outside the crawl scope)
+ * 2. Injects authentication credentials from matching scope URLs
+ * 3. Reconstructs the `withoutHash` URL with injected auth
+ * 4. In recursive mode: enqueues internal lower-layer URLs for full scraping,
+ *    and external URLs for metadata-only scraping (if `fetchExternal` is enabled)
+ * 5. In non-recursive mode: enqueues all URLs for metadata-only scraping
+ * @param anchors - The list of anchor data extracted from the page.
+ * @param scope - Map of hostnames to their scope URLs.
+ * @param options - Crawler configuration options.
+ * @param addUrl - Callback to enqueue a newly discovered URL. Accepts optional
+ *   `{ metadataOnly: true }` to request metadata-only scraping.
+ */
+function processAnchors(anchors, scope, options, addUrl) {
+    for (const anchor of anchors) {
+        const isExternal = isExternalUrl(anchor.href, scope);
+        anchor.isExternal = isExternal;
+        if (!isExternal && (!anchor.href.username || !anchor.href.password)) {
+            injectScopeAuth(anchor.href, scope);
+            const auth = anchor.href.username && anchor.href.password
+                ? `${anchor.href.username}:${anchor.href.password}@`
+                : '';
+            const host = anchor.href.hostname + (anchor.href.port ? `:${anchor.href.port}` : '');
+            const newSearch = anchor.href.query ? `?${anchor.href.query}` : '';
+            const body = anchor.href.dirname
+                ? `${anchor.href.paths.join('/')}${newSearch}`
+                : newSearch
+                    ? `${newSearch}`
+                    : '';
+            const withoutHash = `${anchor.href.protocol}//${auth}${host}${body ? `/${body}` : ''}`;
+            anchor.href.withoutHash = withoutHash;
+        }
+        if (options.recursive) {
+            const scopes = scope.get(anchor.href.hostname);
+            if (scopes && isInAnyLowerLayer(anchor.href, scopes, options)) {
+                addUrl(anchor.href);
+            }
+            else if (isExternal && options.fetchExternal) {
+                addUrl(anchor.href, { metadataOnly: true });
+            }
+            continue;
+        }
+        addUrl(anchor.href, { metadataOnly: true });
+    }
+}

package/lib/crawler/handle-scrape-error.d.ts

@@ -0,0 +1,37 @@
+import type LinkList from './link-list.js';
+import type { CrawlerOptions } from './types.js';
+import type { Link, PageData } from '../utils/index.js';
+import type { ExURL } from '@d-zero/shared/parse-url';
+/**
+ * Handle an error that occurred during page scraping.
+ *
+ * Marks the URL as done and creates a fallback {@link PageData} from the
+ * link, regardless of whether the error caused a shutdown. This ensures
+ * that errored URLs are recorded in the DB (`status = -1, scraped = 1`)
+ * and not re-queued on resume.
+ * @param payload - The error payload from the scraper.
+ * @param payload.url - The URL being scraped when the error occurred, or `null`.
+ * @param payload.error - The error details including name, message, and optional stack.
+ * @param payload.error.name
+ * @param payload.error.message
+ * @param payload.error.stack
+ * @param payload.shutdown - Whether the error caused the scraper process to shut down.
+ * @param payload.pid - The process ID of the scraper, or `undefined`.
+ * @param linkList - The link list managing the crawl queue.
+ * @param scope - Map of hostnames to their scope URLs.
+ * @param options - Crawler configuration options.
+ * @returns An object with the link and an optional fallback PageData result.
+ */
+export declare function handleScrapeError(payload: {
+    url: ExURL | null;
+    error: {
+        name: string;
+        message: string;
+        stack?: string;
+    };
+    shutdown: boolean;
+    pid: number | undefined;
+}, linkList: LinkList, scope: ReadonlyMap<string, readonly ExURL[]>, options: CrawlerOptions): {
+    link: Link | null;
+    result?: PageData;
+};

package/lib/crawler/handle-scrape-error.js

@@ -0,0 +1,38 @@
+import { crawlerErrorLog } from '../debug.js';
+import { linkToPageData } from './link-to-page-data.js';
+/**
+ * Handle an error that occurred during page scraping.
+ *
+ * Marks the URL as done and creates a fallback {@link PageData} from the
+ * link, regardless of whether the error caused a shutdown. This ensures
+ * that errored URLs are recorded in the DB (`status = -1, scraped = 1`)
+ * and not re-queued on resume.
+ * @param payload - The error payload from the scraper.
+ * @param payload.url - The URL being scraped when the error occurred, or `null`.
+ * @param payload.error - The error details including name, message, and optional stack.
+ * @param payload.error.name
+ * @param payload.error.message
+ * @param payload.error.stack
+ * @param payload.shutdown - Whether the error caused the scraper process to shut down.
+ * @param payload.pid - The process ID of the scraper, or `undefined`.
+ * @param linkList - The link list managing the crawl queue.
+ * @param scope - Map of hostnames to their scope URLs.
+ * @param options - Crawler configuration options.
+ * @returns An object with the link and an optional fallback PageData result.
+ */
+export function handleScrapeError(payload, linkList, scope, options) {
+    const { url, error, shutdown, pid } = payload;
+    let link = null;
+    let result;
+    if (url) {
+        const updated = linkList.done(url, scope, { error }, options);
+        if (updated) {
+            link = updated;
+            result = linkToPageData(updated);
+        }
+    }
+    crawlerErrorLog('From %d(%s)', pid, url?.href ?? 'UNKNOWN_URL');
+    crawlerErrorLog('Then shutdown?: %s', shutdown ? 'Yes' : 'No');
+    crawlerErrorLog('%O', error);
+    return { link, result };
+}

package/lib/crawler/index.d.ts

@@ -0,0 +1,2 @@
+export { default as Crawler } from './crawler.js';
+export { clearDestinationCache } from './clear-destination-cache.js';

package/lib/crawler/index.js

@@ -0,0 +1,2 @@
+export { default as Crawler } from './crawler.js';
+export { clearDestinationCache } from './clear-destination-cache.js';

package/lib/crawler/inject-scope-auth.d.ts

@@ -0,0 +1,11 @@
+import type { ExURL } from '@d-zero/shared/parse-url';
+/**
+ * Inject authentication credentials from a matching scope URL into the target URL.
+ *
+ * Finds the best-matching scope URL (deepest path match) for the given URL's
+ * hostname and copies its `username` and `password` properties. This mutates
+ * the `url` parameter in place.
+ * @param url - The parsed URL to receive authentication credentials (mutated in place).
+ * @param scope - Map of hostnames to their scope URLs.
+ */
+export declare function injectScopeAuth(url: ExURL, scope: ReadonlyMap<string, readonly ExURL[]>): void;

package/lib/crawler/inject-scope-auth.js

@@ -0,0 +1,21 @@
+import { findBestMatchingScope } from './find-best-matching-scope.js';
+/**
+ * Inject authentication credentials from a matching scope URL into the target URL.
+ *
+ * Finds the best-matching scope URL (deepest path match) for the given URL's
+ * hostname and copies its `username` and `password` properties. This mutates
+ * the `url` parameter in place.
+ * @param url - The parsed URL to receive authentication credentials (mutated in place).
+ * @param scope - Map of hostnames to their scope URLs.
+ */
+export function injectScopeAuth(url, scope) {
+    const scopes = scope.get(url.hostname);
+    if (!scopes) {
+        return;
+    }
+    const matchedScope = findBestMatchingScope(url, scopes);
+    if (matchedScope) {
+        url.username = matchedScope.username;
+        url.password = matchedScope.password;
+    }
+}

package/lib/crawler/is-external-url.d.ts

@@ -0,0 +1,11 @@
+import type { ExURL } from '@d-zero/shared/parse-url';
+/**
+ * Determine whether a URL is external to the crawl scope.
+ *
+ * A URL is considered external if its hostname does not appear
+ * as a key in the scope map.
+ * @param url - The parsed URL to check.
+ * @param scope - Map of hostnames to their scope URLs.
+ * @returns `true` if the URL is outside the crawl scope.
+ */
+export declare function isExternalUrl(url: ExURL, scope: ReadonlyMap<string, readonly ExURL[]>): boolean;

package/lib/crawler/is-external-url.js

@@ -0,0 +1,12 @@
+/**
+ * Determine whether a URL is external to the crawl scope.
+ *
+ * A URL is considered external if its hostname does not appear
+ * as a key in the scope map.
+ * @param url - The parsed URL to check.
+ * @param scope - Map of hostnames to their scope URLs.
+ * @returns `true` if the URL is outside the crawl scope.
+ */
+export function isExternalUrl(url, scope) {
+    return !scope.has(url.hostname);
+}

package/lib/crawler/is-in-any-lower-layer.d.ts

@@ -0,0 +1,13 @@
+import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
+/**
+ * Check whether a URL is in a lower layer (subdirectory) of any scope URL.
+ *
+ * Tests the URL against each scope URL using the `isLowerLayer` utility,
+ * which checks if the URL's path is at the same level or deeper than
+ * the scope URL's path.
+ * @param url - The parsed URL to check.
+ * @param scopes - The list of scope URLs to test against.
+ * @param options - URL parsing options used for layer comparison.
+ * @returns `true` if the URL is in a lower layer of at least one scope URL.
+ */
+export declare function isInAnyLowerLayer(url: ExURL, scopes: readonly ExURL[], options: ParseURLOptions): boolean;

package/lib/crawler/is-in-any-lower-layer.js

@@ -0,0 +1,15 @@
+import { isLowerLayer } from '@d-zero/shared/is-lower-layer';
+/**
+ * Check whether a URL is in a lower layer (subdirectory) of any scope URL.
+ *
+ * Tests the URL against each scope URL using the `isLowerLayer` utility,
+ * which checks if the URL's path is at the same level or deeper than
+ * the scope URL's path.
+ * @param url - The parsed URL to check.
+ * @param scopes - The list of scope URLs to test against.
+ * @param options - URL parsing options used for layer comparison.
+ * @returns `true` if the URL is in a lower layer of at least one scope URL.
+ */
+export function isInAnyLowerLayer(url, scopes, options) {
+    return scopes.some((scope) => isLowerLayer(url.href, scope.href, options));
+}

package/lib/crawler/link-list.d.ts

@@ -0,0 +1,112 @@
+import type { Link, PageData } from '../utils/index.js';
+import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
+/**
+ * Manages the queue of URLs discovered during crawling.
+ *
+ * Tracks URLs across three states: pending (queued but not started),
+ * in-progress (currently being scraped), and done (scraping completed).
+ * Provides deduplication based on `withoutHashAndAuth` normalization
+ * and tracks page completion counts for progress reporting.
+ */
+export default class LinkList {
+    #private;
+    /**
+     * The number of successfully completed internal HTML pages.
+     *
+     * Only counts pages that are internal, in a lower layer, use HTTP(S),
+     * have no error status, and have `text/html` content type.
+     */
+    get completePages(): number;
+    /**
+     * Add a URL to the pending queue if it has not been seen before.
+     *
+     * Deduplication is based on the URL's `withoutHashAndAuth` representation.
+     * If the URL is already pending, in progress, or done, this is a no-op.
+     * @param linkUrl - The parsed URL to add to the queue.
+     * @param options - Optional flags for the URL.
+     * @param options.metadataOnly - If `true`, marks this URL for title-only scraping
+     *   (metadata extraction without full page processing).
+     * @param options.predicted - If `true`, marks this URL as a predicted pagination guess
+     *   that should be discarded if it returns a 4xx/5xx status.
+     */
+    add(linkUrl: ExURL, options?: {
+        metadataOnly?: true;
+        predicted?: true;
+    }): void;
+    /**
+     * Mark a URL as completed and record its scrape result.
+     *
+     * Moves the URL from pending/progress to done, constructs a {@link Link}
+     * object with scope and layer information, and increments the page counter
+     * if the result qualifies as a valid HTML page.
+     * @param url - The URL that has been scraped.
+     * @param scope - The current scope map (hostname to scope URLs).
+     * @param resource - The scrape result containing page data and/or error information.
+     * @param resource.page - The scraped page data, if the scrape succeeded.
+     * @param resource.error - The error object, if the scrape failed.
+     * @param options - URL parsing options (e.g., `disableQueries`).
+     * @returns The constructed {@link Link} object, or `null` if the URL was not in the queue.
+     */
+    done(url: ExURL, scope: ReadonlyMap<string, readonly ExURL[]>, resource: {
+        page?: PageData;
+        error?: Error;
+    }, options: ParseURLOptions): Link | null;
+    /**
+     * Get the current pending and in-progress URL lists.
+     * @returns An object containing arrays of pending and in-progress URL strings.
+     */
+    getLinks(): {
+        /** URLs queued but not yet started. */
+        pending: string[];
+        /** URLs currently being scraped. */
+        progress: string[];
+    };
+    /**
+     * Get a summary of crawl progress counts.
+     * @returns An object with total/completed counts for both all links and pages only.
+     */
+    getPageCount(): {
+        /** Total number of discovered links (pending + progress + done). */
+        totalLinks: number;
+        /** Number of links that have been fully processed. */
+        completedLinks: number;
+        /** Total number of discovered pages (pending + progress + completed pages). */
+        totalPages: number;
+        /** Number of pages that have been successfully scraped. */
+        completedPages: number;
+    };
+    /**
+     * Check whether a URL is flagged for title-only scraping.
+     *
+     * Title-only scraping extracts only the page title and basic metadata,
+     * without processing anchors or capturing the full HTML.
+     * @param urlWithoutHashAndAuth - The normalized URL string (without hash and auth) to check.
+     * @returns `true` if the URL should be scraped in title-only mode.
+     */
+    isMetadataOnly(urlWithoutHashAndAuth: string): boolean;
+    /**
+     * Check whether a URL was added as a predicted pagination URL.
+     * @param urlWithoutHashAndAuth - The normalized URL string (without hash and auth) to check.
+     * @returns `true` if the URL was added with the predicted flag.
+     */
+    isPredicted(urlWithoutHashAndAuth: string): boolean;
+    /**
+     * Transition a URL from the pending state to the in-progress state.
+     *
+     * This should be called when scraping of the URL actually begins.
+     * If the URL is not in the pending set, this is a no-op.
+     * @param url - The URL that is now being actively scraped.
+     */
+    progress(url: ExURL): void;
+    /**
+     * Restore the link list state from a previous crawl session.
+     *
+     * Re-adds pending URLs to the queue and marks previously done URLs
+     * as completed, enabling the crawler to resume from where it left off.
+     * @param pending - URLs that were pending in the previous session.
+     * @param done - URLs that were already completed in the previous session.
+     * @param options - URL parsing options for re-parsing the pending URLs.
+     * @returns The parsed pending URLs that were successfully added to the queue.
+     */
+    resume(pending: string[], done: string[], options: ParseURLOptions): ExURL[];
+}