@nitpicker/crawler 0.9.0 → 0.12.0

package/lib/crawler/create-change-phase-handler.d.ts

@@ -0,0 +1,54 @@
+import type { BufferedPhaseError } from './drain-phase-errors.js';
+import type { ChangePhaseEvent } from '@d-zero/beholder';
+/**
+ * Options for {@link createChangePhaseHandler}. Declared structurally so
+ * tests can pass plain `vi.fn()` stubs without spinning up a real Crawler.
+ */
+export interface ChangePhaseHandlerOptions {
+    /**
+     * Forwards the raw {@link ChangePhaseEvent} so external listeners on the
+     * Crawler still see every transition (typically `this.emit.bind(this)`
+     * narrowed to the `changePhase` channel).
+     */
+    emit: (event: ChangePhaseEvent) => void;
+    /** Receives the formatted progress log line. Skipped when empty. */
+    update: (log: string) => void;
+    /**
+     * Renders the human-readable progress message for an event. Injected so
+     * the handler stays free of the Crawler's internal log formatter.
+     * Returns `null` for events that should not surface to `update`.
+     */
+    formatLog: (event: ChangePhaseEvent) => string | null;
+    /**
+     * Per-URL buffer of `retryExhausted` failures. The handler appends to
+     * this map; it does not drain (that is `drainPhaseErrors`'s job).
+     */
+    buffer: Map<string, BufferedPhaseError[]>;
+    /** URL href used as the buffer key for this scrape. */
+    urlHref: string;
+}
+/**
+ * Builds the `scraper.on('changePhase', ...)` listener used by
+ * {@link Crawler.#launchBrowserAndScrape}.
+ *
+ * Three responsibilities:
+ * 1. Render the phase log via the injected `formatLog` and pipe it to `update`.
+ * 2. Forward the raw event so external consumers (CLI progress UI etc.) see
+ *    every transition.
+ * 3. Buffer `retryExhausted` events into the per-URL phase-error map so they
+ *    can be drained as `pageError` events AFTER the `page` event fires.
+ *
+ * WHY a factory: the listener captures per-scrape state (`buffer`, `urlHref`,
+ * `update`). Extracting the factory makes the wiring directly unit-testable
+ * with plain stubs, instead of requiring a mocked Puppeteer + beholder + dealer
+ * stack to drive the worker.
+ *
+ * **Caller contract**: register the returned handler at most once per
+ * `scraper` instance. The Crawler creates a fresh Scraper per URL so this
+ * holds today; if scraper pooling is ever introduced, register exactly one
+ * handler per scrape and unregister it on completion to avoid duplicate
+ * buffer entries.
+ * @param options - Wiring dependencies for the handler.
+ * @returns A function suitable for `scraper.on('changePhase', ...)`.
+ */
+export declare function createChangePhaseHandler(options: ChangePhaseHandlerOptions): (event: ChangePhaseEvent) => void;

package/lib/crawler/create-change-phase-handler.js

@@ -0,0 +1,44 @@
+/**
+ * Builds the `scraper.on('changePhase', ...)` listener used by
+ * {@link Crawler.#launchBrowserAndScrape}.
+ *
+ * Three responsibilities:
+ * 1. Render the phase log via the injected `formatLog` and pipe it to `update`.
+ * 2. Forward the raw event so external consumers (CLI progress UI etc.) see
+ *    every transition.
+ * 3. Buffer `retryExhausted` events into the per-URL phase-error map so they
+ *    can be drained as `pageError` events AFTER the `page` event fires.
+ *
+ * WHY a factory: the listener captures per-scrape state (`buffer`, `urlHref`,
+ * `update`). Extracting the factory makes the wiring directly unit-testable
+ * with plain stubs, instead of requiring a mocked Puppeteer + beholder + dealer
+ * stack to drive the worker.
+ *
+ * **Caller contract**: register the returned handler at most once per
+ * `scraper` instance. The Crawler creates a fresh Scraper per URL so this
+ * holds today; if scraper pooling is ever introduced, register exactly one
+ * handler per scrape and unregister it on completion to avoid duplicate
+ * buffer entries.
+ * @param options - Wiring dependencies for the handler.
+ * @returns A function suitable for `scraper.on('changePhase', ...)`.
+ */
+export function createChangePhaseHandler(options) {
+    const { emit, update, formatLog, buffer, urlHref } = options;
+    return (event) => {
+        const msg = formatLog(event);
+        if (msg) {
+            update(msg);
+        }
+        emit(event);
+        // retryExhausted fires when beholder's @retryable gives up on a
+        // secondary scrape step (e.g. a viewport switch detaching the frame
+        // in #fetchImages). The page itself still completes, so we buffer
+        // the failure here and emit it as a pageError after the page event
+        // has been emitted.
+        if (event.name === 'retryExhausted') {
+            const list = buffer.get(urlHref) ?? [];
+            list.push({ phase: event.name, message: event.message });
+            buffer.set(urlHref, list);
+        }
+    };
+}

package/lib/crawler/derive-js-redirect-target.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Decide whether a puppeteer post-navigation URL represents a real
+ * client-side redirect (JS / meta-refresh) or just noise.
+ *
+ * **Why this helper exists:** when `page.goto()` returns `null`, the upstream
+ * scraper throws `The method Page.goto returned null` — classified as
+ * `protocol`, neither permanent nor a puppeteer-fallback kind, so
+ * `--retry-failed` never converges. Reading `page.url()` after the throw is
+ * the only way to recover the destination Chromium actually navigated to (the
+ * thrown error carries no URL), so we can record the source as a redirect
+ * edge instead of a hard `status = -1`. But `page.url()` can also report
+ * uninformative values — `about:blank` before the first navigation completes,
+ * the original URL when nothing happened — and a naive "different ?" check
+ * would create a phantom redirect every time the navigation simply failed at
+ * the same URL.
+ *
+ * The filter is intentionally narrow: anything that does not look like an
+ * `http(s):` URL semantically distinct from the originally requested
+ * location is discarded. Edge cases the test pins:
+ *
+ * - identity after URL canonicalisation (case / trailing-slash / default
+ *   port / credentials / fragment) → null. WHATWG URL parsing handles
+ *   `https://Example.COM` vs `https://example.com/`, `https://host:443/`
+ *   vs `https://host/`, `https://u:p@host/x` vs `https://host/x`, etc.
+ * - `about:blank` / `chrome-error://...` / `data:` / `file:` → null
+ *   (browser-internal sentinels, never a legitimate destination)
+ * - empty / whitespace string → null (defensive against
+ *   `page.url()` returning `''` on a brand-new context)
+ * - any genuine `http(s):` URL semantically different from the source →
+ *   returned credential-/fragment-stripped (defence-in-depth against a
+ *   pre-RFC server issuing `Location: https://user:pass@host/path`, which
+ *   would otherwise persist credentials into the `.nitpicker` archive
+ *   — exactly the leak class the scope-auth-leak guard at the navigation
+ *   side blocks)
+ *
+ * The caller does NOT need to pre-normalise URLs — this helper normalises
+ * both sides via the WHATWG URL parser before comparing. The production
+ * call site passes `url.withoutHashAndAuth` for `originalUrl`, but the
+ * normalisation here is idempotent so passing `url.href` would also work.
+ * @param originalUrl - The URL puppeteer was asked to navigate to.
+ * @param postNavigationUrl - The URL reported by `page.url()` after the
+ *   throw. May be `null` / `undefined` when reading the URL itself failed.
+ * @returns The credential-/fragment-stripped post-navigation URL when it
+ *   represents a real JS redirect, otherwise `null`.
+ * @example
+ * ```ts
+ * deriveJsRedirectTarget(
+ *   'https://www.example.com/old',
+ *   'https://www.example.com/new',
+ * ); // → 'https://www.example.com/new'
+ *
+ * deriveJsRedirectTarget('https://www.example.com/old', 'about:blank');
+ * // → null
+ *
+ * // Case / trailing-slash noise — no phantom self-redirect:
+ * deriveJsRedirectTarget(
+ *   'https://www.example.com',
+ *   'https://www.example.com/',
+ * ); // → null
+ *
+ * // Credentials in destination — stripped before return:
+ * deriveJsRedirectTarget(
+ *   'https://www.example.com/',
+ *   'https://user:pass@www.example.com/dest',
+ * ); // → 'https://www.example.com/dest'
+ * ```
+ */
+export declare function deriveJsRedirectTarget(originalUrl: string, postNavigationUrl: string | null | undefined): string | null;

package/lib/crawler/derive-js-redirect-target.js

@@ -0,0 +1,129 @@
+/** Matches only the `http:` and `https:` schemes (case-insensitive). */
+const HTTP_SCHEME_PATTERN = /^https?:\/\//i;
+/**
+ * Canonicalise a URL for the JS-redirect identity check and return value:
+ * strip credentials and fragment, normalise case / default-ports /
+ * trailing-slash via the WHATWG URL parser. Returns `null` when the input
+ * is unparseable.
+ *
+ * Used for both sides of the comparison so case-only or trailing-slash-only
+ * differences (`https://Example.com` vs `https://example.com/`) do not
+ * produce phantom self-redirects, and for the returned target so a pre-RFC
+ * `Location: https://user:pass@host/path` is never persisted into the
+ * archive with credentials (same threat model the scope-auth-leak guard
+ * mitigates on the navigation side).
+ * @param rawUrl - The URL string to canonicalise.
+ * @returns The canonical URL string, or `null` when unparseable.
+ */
+function canonicaliseForComparison(rawUrl) {
+    try {
+        const parsed = new URL(rawUrl);
+        parsed.username = '';
+        parsed.password = '';
+        parsed.hash = '';
+        return parsed.href;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Decide whether a puppeteer post-navigation URL represents a real
+ * client-side redirect (JS / meta-refresh) or just noise.
+ *
+ * **Why this helper exists:** when `page.goto()` returns `null`, the upstream
+ * scraper throws `The method Page.goto returned null` — classified as
+ * `protocol`, neither permanent nor a puppeteer-fallback kind, so
+ * `--retry-failed` never converges. Reading `page.url()` after the throw is
+ * the only way to recover the destination Chromium actually navigated to (the
+ * thrown error carries no URL), so we can record the source as a redirect
+ * edge instead of a hard `status = -1`. But `page.url()` can also report
+ * uninformative values — `about:blank` before the first navigation completes,
+ * the original URL when nothing happened — and a naive "different ?" check
+ * would create a phantom redirect every time the navigation simply failed at
+ * the same URL.
+ *
+ * The filter is intentionally narrow: anything that does not look like an
+ * `http(s):` URL semantically distinct from the originally requested
+ * location is discarded. Edge cases the test pins:
+ *
+ * - identity after URL canonicalisation (case / trailing-slash / default
+ *   port / credentials / fragment) → null. WHATWG URL parsing handles
+ *   `https://Example.COM` vs `https://example.com/`, `https://host:443/`
+ *   vs `https://host/`, `https://u:p@host/x` vs `https://host/x`, etc.
+ * - `about:blank` / `chrome-error://...` / `data:` / `file:` → null
+ *   (browser-internal sentinels, never a legitimate destination)
+ * - empty / whitespace string → null (defensive against
+ *   `page.url()` returning `''` on a brand-new context)
+ * - any genuine `http(s):` URL semantically different from the source →
+ *   returned credential-/fragment-stripped (defence-in-depth against a
+ *   pre-RFC server issuing `Location: https://user:pass@host/path`, which
+ *   would otherwise persist credentials into the `.nitpicker` archive
+ *   — exactly the leak class the scope-auth-leak guard at the navigation
+ *   side blocks)
+ *
+ * The caller does NOT need to pre-normalise URLs — this helper normalises
+ * both sides via the WHATWG URL parser before comparing. The production
+ * call site passes `url.withoutHashAndAuth` for `originalUrl`, but the
+ * normalisation here is idempotent so passing `url.href` would also work.
+ * @param originalUrl - The URL puppeteer was asked to navigate to.
+ * @param postNavigationUrl - The URL reported by `page.url()` after the
+ *   throw. May be `null` / `undefined` when reading the URL itself failed.
+ * @returns The credential-/fragment-stripped post-navigation URL when it
+ *   represents a real JS redirect, otherwise `null`.
+ * @example
+ * ```ts
+ * deriveJsRedirectTarget(
+ *   'https://www.example.com/old',
+ *   'https://www.example.com/new',
+ * ); // → 'https://www.example.com/new'
+ *
+ * deriveJsRedirectTarget('https://www.example.com/old', 'about:blank');
+ * // → null
+ *
+ * // Case / trailing-slash noise — no phantom self-redirect:
+ * deriveJsRedirectTarget(
+ *   'https://www.example.com',
+ *   'https://www.example.com/',
+ * ); // → null
+ *
+ * // Credentials in destination — stripped before return:
+ * deriveJsRedirectTarget(
+ *   'https://www.example.com/',
+ *   'https://user:pass@www.example.com/dest',
+ * ); // → 'https://www.example.com/dest'
+ * ```
+ */
+export function deriveJsRedirectTarget(originalUrl, postNavigationUrl) {
+    if (typeof postNavigationUrl !== 'string') {
+        return null;
+    }
+    const trimmed = postNavigationUrl.trim();
+    if (trimmed === '') {
+        return null;
+    }
+    // Browser-internal sentinels: `about:blank` appears before the first
+    // navigation completes, `chrome-error://...` after a network error
+    // renders the Chromium error page, and `data:` / `file:` / `javascript:`
+    // can never be the destination of an off-page redirect we want to record.
+    // `http:` / `https:` is the only safe positive match.
+    if (!HTTP_SCHEME_PATTERN.test(trimmed)) {
+        return null;
+    }
+    const destinationCanonical = canonicaliseForComparison(trimmed);
+    if (destinationCanonical === null) {
+        return null;
+    }
+    const originalCanonical = canonicaliseForComparison(originalUrl);
+    // `originalCanonical === null` only happens for a degenerate (unparseable)
+    // `originalUrl` — fall back to a raw string check so we never claim a
+    // redirect we cannot prove. The production call site always supplies a
+    // parseable form.
+    if (originalCanonical === null) {
+        return trimmed === originalUrl ? null : destinationCanonical;
+    }
+    if (destinationCanonical === originalCanonical) {
+        return null;
+    }
+    return destinationCanonical;
+}

package/lib/crawler/derive-page-source.d.ts

@@ -0,0 +1,23 @@
+import type { InventoryMode } from './types.js';
+import type { PageSource } from '../archive/types.js';
+/**
+ * Decide which {@link PageSource} label a newly-scraped page row should carry.
+ *
+ * When the crawler is NOT in inventory mode (`inventoryMode === null`),
+ * returns `undefined` — the caller emits no `source` and the DB DEFAULT
+ * `'crawled'` ends up on the row. This keeps the normal crawl path
+ * untouched.
+ *
+ * When inventory mode is active, the URL is matched against
+ * `inventoryMode.seedUrls`. A hit means the URL came straight from the
+ * user-supplied list (`'inventory-seed'`); a miss means the URL was found
+ * by following links from a seed page (`'inventory-discovered'`).
+ *
+ * Sub-resources captured by puppeteer during inventory-mode rendering are
+ * NEVER seeds — the caller for those events always passes
+ * `'inventory-discovered'` directly without consulting this helper.
+ * @param inventoryMode - Inventory-mode config from `CrawlerOptions.inventoryMode`, or `null` outside `--inventory`.
+ * @param pageUrlWithoutHashAndAuth - The page URL keyed by `withoutHashAndAuth` (auth credentials stripped, hash dropped).
+ * @returns The label to write to `pages.source`, or `undefined` for the DB default.
+ */
+export declare function derivePageSource(inventoryMode: InventoryMode | null, pageUrlWithoutHashAndAuth: string): PageSource | undefined;

package/lib/crawler/derive-page-source.js

@@ -0,0 +1,28 @@
+/**
+ * Decide which {@link PageSource} label a newly-scraped page row should carry.
+ *
+ * When the crawler is NOT in inventory mode (`inventoryMode === null`),
+ * returns `undefined` — the caller emits no `source` and the DB DEFAULT
+ * `'crawled'` ends up on the row. This keeps the normal crawl path
+ * untouched.
+ *
+ * When inventory mode is active, the URL is matched against
+ * `inventoryMode.seedUrls`. A hit means the URL came straight from the
+ * user-supplied list (`'inventory-seed'`); a miss means the URL was found
+ * by following links from a seed page (`'inventory-discovered'`).
+ *
+ * Sub-resources captured by puppeteer during inventory-mode rendering are
+ * NEVER seeds — the caller for those events always passes
+ * `'inventory-discovered'` directly without consulting this helper.
+ * @param inventoryMode - Inventory-mode config from `CrawlerOptions.inventoryMode`, or `null` outside `--inventory`.
+ * @param pageUrlWithoutHashAndAuth - The page URL keyed by `withoutHashAndAuth` (auth credentials stripped, hash dropped).
+ * @returns The label to write to `pages.source`, or `undefined` for the DB default.
+ */
+export function derivePageSource(inventoryMode, pageUrlWithoutHashAndAuth) {
+    if (inventoryMode === null) {
+        return undefined;
+    }
+    return inventoryMode.seedUrls.has(pageUrlWithoutHashAndAuth)
+        ? 'inventory-seed'
+        : 'inventory-discovered';
+}

package/lib/crawler/derive-resource-source.d.ts

@@ -0,0 +1,33 @@
+import type { PageSource } from '../archive/types.js';
+/**
+ * Decide which {@link PageSource} label a newly-captured sub-resource row
+ * (CSS / JS / image / fetch response) should carry, based on the lineage of
+ * the page that is currently being rendered.
+ *
+ * Sub-resources are NEVER themselves seeds — even when puppeteer is
+ * rendering a page that *is* an inventory seed, the assets it pulls in are
+ * downstream and must be labelled `'inventory-discovered'`. The seed label
+ * is reserved for URLs that were explicitly handed in by the user via the
+ * `--inventory` file. Likewise, when a sub-resource is captured during a
+ * render of an `'inventory-discovered'` page (a page reached transitively
+ * through the inventory chain), the asset is still inventory-discovered —
+ * not a new seed.
+ *
+ * Outside the inventory chain (parent is `'crawled'` or has no source
+ * record) this returns `undefined` so the caller emits no `source` and the
+ * DB DEFAULT (`'crawled'`) lands on the row, leaving the normal crawl path
+ * untouched.
+ *
+ * `parentSource` must reflect the MERGED source of the page being scraped:
+ * for an active `--inventory` session that comes from
+ * {@link import('./derive-page-source.js').derivePageSource}, for a
+ * `--resume` / `--retry-failed` session it comes from a DB lookup
+ * (`PageSourceLookup`) because `inventoryMode` is not persisted across
+ * sessions. The two-stage resolution is what keeps sub-resource labels
+ * correct on resume: even though the orchestrator no longer carries the
+ * inventory seed set in memory, the DB still records the parent's lineage
+ * and we propagate it forward.
+ * @param parentSource - Merged source of the page that is producing this sub-resource (or `undefined` for an unknown / `'crawled'` parent).
+ * @returns The label to write to `resources.source`, or `undefined` for the DB default.
+ */
+export declare function deriveResourceSource(parentSource: PageSource | undefined): PageSource | undefined;

package/lib/crawler/derive-resource-source.js

@@ -0,0 +1,37 @@
+/**
+ * Decide which {@link PageSource} label a newly-captured sub-resource row
+ * (CSS / JS / image / fetch response) should carry, based on the lineage of
+ * the page that is currently being rendered.
+ *
+ * Sub-resources are NEVER themselves seeds — even when puppeteer is
+ * rendering a page that *is* an inventory seed, the assets it pulls in are
+ * downstream and must be labelled `'inventory-discovered'`. The seed label
+ * is reserved for URLs that were explicitly handed in by the user via the
+ * `--inventory` file. Likewise, when a sub-resource is captured during a
+ * render of an `'inventory-discovered'` page (a page reached transitively
+ * through the inventory chain), the asset is still inventory-discovered —
+ * not a new seed.
+ *
+ * Outside the inventory chain (parent is `'crawled'` or has no source
+ * record) this returns `undefined` so the caller emits no `source` and the
+ * DB DEFAULT (`'crawled'`) lands on the row, leaving the normal crawl path
+ * untouched.
+ *
+ * `parentSource` must reflect the MERGED source of the page being scraped:
+ * for an active `--inventory` session that comes from
+ * {@link import('./derive-page-source.js').derivePageSource}, for a
+ * `--resume` / `--retry-failed` session it comes from a DB lookup
+ * (`PageSourceLookup`) because `inventoryMode` is not persisted across
+ * sessions. The two-stage resolution is what keeps sub-resource labels
+ * correct on resume: even though the orchestrator no longer carries the
+ * inventory seed set in memory, the DB still records the parent's lineage
+ * and we propagate it forward.
+ * @param parentSource - Merged source of the page that is producing this sub-resource (or `undefined` for an unknown / `'crawled'` parent).
+ * @returns The label to write to `resources.source`, or `undefined` for the DB default.
+ */
+export function deriveResourceSource(parentSource) {
+    if (parentSource === 'inventory-seed' || parentSource === 'inventory-discovered') {
+        return 'inventory-discovered';
+    }
+    return undefined;
+}

package/lib/crawler/dns-burned-host-cache.d.ts

@@ -0,0 +1,26 @@
+import type { ErrorKind } from '../types.js';
+/**
+ * In-memory set of hostnames known to be unreachable due to DNS errors
+ * (e.g. `getaddrinfo ENOTFOUND`).
+ *
+ * Lives for one crawl session — cleared by {@link clearDnsBurnedHostCache} at
+ * the same four orchestrator sites as {@link destinationCache}. Hosts are
+ * marked in two ways:
+ *
+ * - **Session learning**: the `onGiveUp` callback of `#sendHeadRequest`
+ *   classifies the final retry error and stores `'dns'` when the matcher
+ *   fires. The next URL on the same hostname short-circuits before retry.
+ * - **Session preload**: re-open paths (`append` / `inventory` / `retryFailed`
+ *   / `resume`) call {@link Archive.listDnsBurnedHostCandidates} and seed the
+ *   map from `crawl_errors`, so previously-burned hosts cost zero retries on
+ *   the next crawl.
+ *
+ * Keys are always `url.hostname.toLowerCase()` — WHATWG URL has already
+ * Punycoded IDNs and stripped the port, so no extra normalization is needed
+ * for IPv4 / IPv6 literals or international hostnames.
+ *
+ * The value records the originating {@link ErrorKind}. Only `'dns'` is set
+ * today; the union shape is preserved so future error classes
+ * (e.g. `tls`, `connection-refused`) can extend the same cache.
+ */
+export declare const dnsBurnedHostCache: Map<string, ErrorKind>;

package/lib/crawler/dns-burned-host-cache.js

@@ -0,0 +1,25 @@
+/**
+ * In-memory set of hostnames known to be unreachable due to DNS errors
+ * (e.g. `getaddrinfo ENOTFOUND`).
+ *
+ * Lives for one crawl session — cleared by {@link clearDnsBurnedHostCache} at
+ * the same four orchestrator sites as {@link destinationCache}. Hosts are
+ * marked in two ways:
+ *
+ * - **Session learning**: the `onGiveUp` callback of `#sendHeadRequest`
+ *   classifies the final retry error and stores `'dns'` when the matcher
+ *   fires. The next URL on the same hostname short-circuits before retry.
+ * - **Session preload**: re-open paths (`append` / `inventory` / `retryFailed`
+ *   / `resume`) call {@link Archive.listDnsBurnedHostCandidates} and seed the
+ *   map from `crawl_errors`, so previously-burned hosts cost zero retries on
+ *   the next crawl.
+ *
+ * Keys are always `url.hostname.toLowerCase()` — WHATWG URL has already
+ * Punycoded IDNs and stripped the port, so no extra normalization is needed
+ * for IPv4 / IPv6 literals or international hostnames.
+ *
+ * The value records the originating {@link ErrorKind}. Only `'dns'` is set
+ * today; the union shape is preserved so future error classes
+ * (e.g. `tls`, `connection-refused`) can extend the same cache.
+ */
+export const dnsBurnedHostCache = new Map();

package/lib/crawler/dns-burned-host-short-circuit-counter.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Session-scoped counter for how many URL fetches were short-circuited by the
+ * {@link dnsBurnedHostCache}. The orchestrator reads this at crawl shutdown
+ * to emit a `[preload] Short-circuited N URL(s)` summary line, then
+ * {@link clearDnsBurnedHostCache} zeroes it for the next session.
+ *
+ * Exposed as a mutable object (not a plain `let`) so that the counter remains
+ * a single shared reference across crawler / orchestrator imports — `let`
+ * bindings cannot be mutated from another module.
+ */
+export declare const dnsBurnedHostShortCircuitCounter: {
+    count: number;
+};

package/lib/crawler/dns-burned-host-short-circuit-counter.js

@@ -0,0 +1,11 @@
+/**
+ * Session-scoped counter for how many URL fetches were short-circuited by the
+ * {@link dnsBurnedHostCache}. The orchestrator reads this at crawl shutdown
+ * to emit a `[preload] Short-circuited N URL(s)` summary line, then
+ * {@link clearDnsBurnedHostCache} zeroes it for the next session.
+ *
+ * Exposed as a mutable object (not a plain `let`) so that the counter remains
+ * a single shared reference across crawler / orchestrator imports — `let`
+ * bindings cannot be mutated from another module.
+ */
+export const dnsBurnedHostShortCircuitCounter = { count: 0 };

package/lib/crawler/drain-phase-errors.d.ts

@@ -0,0 +1,48 @@
+/**
+ * A buffered phase-error record awaiting emission as a `pageError` event.
+ *
+ * `phase` is the beholder phase name (typically `'retryExhausted'`) and
+ * `message` is the human-readable failure text.
+ */
+export interface BufferedPhaseError {
+    /** Scrape phase name. */
+    phase: string;
+    /** Human-readable failure message. */
+    message: string;
+}
+/**
+ * Emitter signature accepted by {@link drainPhaseErrors} for the `pageError`
+ * event. Declared structurally so the Crawler's typed event emitter can be
+ * adapted with a thin closure at the call site without leaking through here.
+ */
+export type DrainPhaseErrorsEmit = (payload: {
+    /** URL of the affected page. */
+    url: string;
+    /** Scrape phase name. */
+    phase: string;
+    /** Human-readable failure message. */
+    message: string;
+    /** Whether the URL is external to the crawl scope. */
+    isExternal: boolean;
+}) => void;
+/**
+ * Drains the buffered phase errors for a URL: removes the entry from
+ * `buffer` and invokes `emit` once per buffered record.
+ *
+ * WHY a standalone function: the Crawler buffers `retryExhausted` events
+ * keyed by `url.href` during scrapeStart, then flushes them as
+ * `pageError` events AFTER `page` / `externalPage` has been emitted so the
+ * orchestrator's WriteQueue serialises `setPage` before `insertPageError`.
+ * Extracting the drain step here makes the flush + delete contract
+ * directly unit-testable without spinning up a real Crawler.
+ *
+ * Idempotent: calling twice for the same `urlHref` is safe — the second
+ * call sees an empty buffer and is a no-op.
+ * @param buffer - The pending-phase-errors map, keyed by URL href.
+ * @param urlHref - URL whose buffered errors should be drained.
+ * @param isExternal - Whether the URL is external to the crawl scope.
+ * @param emit - Callback invoked once per buffered phase-error record.
+ * @returns The number of phase-error events emitted (0 when the buffer
+ *   had no entry for `urlHref`).
+ */
+export declare function drainPhaseErrors(buffer: Map<string, BufferedPhaseError[]>, urlHref: string, isExternal: boolean, emit: DrainPhaseErrorsEmit): number;

package/lib/crawler/drain-phase-errors.js

@@ -0,0 +1,35 @@
+/**
+ * Drains the buffered phase errors for a URL: removes the entry from
+ * `buffer` and invokes `emit` once per buffered record.
+ *
+ * WHY a standalone function: the Crawler buffers `retryExhausted` events
+ * keyed by `url.href` during scrapeStart, then flushes them as
+ * `pageError` events AFTER `page` / `externalPage` has been emitted so the
+ * orchestrator's WriteQueue serialises `setPage` before `insertPageError`.
+ * Extracting the drain step here makes the flush + delete contract
+ * directly unit-testable without spinning up a real Crawler.
+ *
+ * Idempotent: calling twice for the same `urlHref` is safe — the second
+ * call sees an empty buffer and is a no-op.
+ * @param buffer - The pending-phase-errors map, keyed by URL href.
+ * @param urlHref - URL whose buffered errors should be drained.
+ * @param isExternal - Whether the URL is external to the crawl scope.
+ * @param emit - Callback invoked once per buffered phase-error record.
+ * @returns The number of phase-error events emitted (0 when the buffer
+ *   had no entry for `urlHref`).
+ */
+export function drainPhaseErrors(buffer, urlHref, isExternal, emit) {
+    const errors = buffer.get(urlHref);
+    if (!errors || errors.length === 0)
+        return 0;
+    buffer.delete(urlHref);
+    for (const err of errors) {
+        emit({
+            url: urlHref,
+            phase: err.phase,
+            message: err.message,
+            isExternal,
+        });
+    }
+    return errors.length;
+}

package/lib/crawler/fetch-destination.d.ts

@@ -20,19 +20,27 @@ export interface FetchDestinationParams {
     };
     /** User-Agent string to send with the request. */
     readonly userAgent?: string;
+    /**
+     * Race timeout for the network request in milliseconds. Defaults to
+     * {@link DEFAULT_HEAD_TIMEOUT_MS} (10s). `Crawler.#sendHeadRequest` passes
+     * a longer value on later retry attempts so a slow-but-reachable server
+     * gets another chance before being given up on.
+     */
+    readonly timeout?: number;
 }
 /**
  * Fetches the destination metadata for a URL using an HTTP HEAD request (or GET as fallback).
  *
  * Results are cached in memory so that repeated calls for the same URL
- * (without hash) return immediately. The request races against a 10-second
- * timeout; if the server does not respond in time, a {@link NetTimeoutError} is thrown.
+ * (without hash) return immediately. The request races against a configurable
+ * timeout (defaults to {@link DEFAULT_HEAD_TIMEOUT_MS}, 10 seconds); if the
+ * server does not respond in time, a {@link NetTimeoutError} is thrown.
  *
  * If the server returns 405 (Method Not Allowed), 501 (Not Implemented), or 503
  * (Service Unavailable) for a HEAD request, the function automatically retries with GET.
- * @param params - Parameters containing URL, external flag, method, options, and optional User-Agent.
+ * @param params - Parameters containing URL, external flag, method, options, optional User-Agent, and optional timeout.
  * @returns The page metadata obtained from the HTTP response.
- * @throws {NetTimeoutError} If the request exceeds the 10-second timeout.
+ * @throws {NetTimeoutError} If the request exceeds the configured timeout.
  * @throws {Error} If the HTTP request fails for any other reason.
  */
 export declare function fetchDestination(params: FetchDestinationParams): Promise<PageData>;