@nitpicker/crawler 0.11.0 → 0.13.0

package/lib/crawler/is-js-redirect-error-shape.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Decide whether a browser-scrape error message is the specific
+ * `Page.goto() returned null` shape that the JS-redirect rescue is
+ * designed to recover from.
+ *
+ * **Why this gate exists:** before the gate, the rescue fired on *any*
+ * thrown error from `scraper.scrapeStart` as long as `page.url()` happened
+ * to report a different http(s) URL. That made every browser failure
+ * (TLS, target-crashed, OOM, navigation timeout, …) that incidentally
+ * left the page on a follow-up URL look like a JS redirect, hiding the
+ * real failure mode and stamping a phantom `status = 301` on the source.
+ *
+ * The narrow trigger only fires on the upstream's exact sentinel —
+ * `Page.goto returned null` — which beholder's scraper throws *only*
+ * when puppeteer's `page.goto()` resolved to `null`. Substring match (not
+ * equality) so wrapped variants like `[Retried 3 times] The method
+ * Page.goto returned null` (which surface in `crawl_errors` after retry
+ * exhaustion at outer layers) still classify, even though the rescue
+ * sees the bare form. Case-insensitive on the marker so a future
+ * beholder bump that lowercases the message keeps working.
+ *
+ * The trigger keys off the message *string*, not the message-classifier
+ * `kind`, because the rescue runs *before* the kind decision: the kind
+ * classifier would already wash this into `protocol`, and `protocol`
+ * covers more than just goto-null (Target closed / Session closed /
+ * detached Frame …) — none of which leave puppeteer with a meaningful
+ * post-navigation URL to recover.
+ * @param message - The raw error message from
+ *   `BrowserScrapeResult.error.message` (or any string that may carry
+ *   the sentinel inside a wrapper). `null` / `undefined` returns `false`.
+ * @returns `true` iff the message carries the `Page.goto returned null`
+ *   sentinel.
+ * @example
+ * ```ts
+ * isJsRedirectErrorShape('The method Page.goto returned null'); // → true
+ * isJsRedirectErrorShape('Navigation timeout of 60000 ms exceeded'); // → false
+ * isJsRedirectErrorShape(undefined); // → false
+ * ```
+ */
+export declare function isJsRedirectErrorShape(message: string | null | undefined): boolean;

package/lib/crawler/is-js-redirect-error-shape.js

@@ -0,0 +1,53 @@
+/**
+ * Sentinel string emitted by `@d-zero/beholder`'s scraper when
+ * `await page.goto(...)` resolves to `null`. Pinned here because the
+ * JS-redirect rescue in `Crawler.#scrapePage` keys off the exact text —
+ * an upstream rename would silently disable the rescue, but the
+ * spec on this helper would also break, surfacing the drift in CI.
+ */
+const PAGE_GOTO_NULL_MARKER = 'Page.goto returned null';
+/**
+ * Decide whether a browser-scrape error message is the specific
+ * `Page.goto() returned null` shape that the JS-redirect rescue is
+ * designed to recover from.
+ *
+ * **Why this gate exists:** before the gate, the rescue fired on *any*
+ * thrown error from `scraper.scrapeStart` as long as `page.url()` happened
+ * to report a different http(s) URL. That made every browser failure
+ * (TLS, target-crashed, OOM, navigation timeout, …) that incidentally
+ * left the page on a follow-up URL look like a JS redirect, hiding the
+ * real failure mode and stamping a phantom `status = 301` on the source.
+ *
+ * The narrow trigger only fires on the upstream's exact sentinel —
+ * `Page.goto returned null` — which beholder's scraper throws *only*
+ * when puppeteer's `page.goto()` resolved to `null`. Substring match (not
+ * equality) so wrapped variants like `[Retried 3 times] The method
+ * Page.goto returned null` (which surface in `crawl_errors` after retry
+ * exhaustion at outer layers) still classify, even though the rescue
+ * sees the bare form. Case-insensitive on the marker so a future
+ * beholder bump that lowercases the message keeps working.
+ *
+ * The trigger keys off the message *string*, not the message-classifier
+ * `kind`, because the rescue runs *before* the kind decision: the kind
+ * classifier would already wash this into `protocol`, and `protocol`
+ * covers more than just goto-null (Target closed / Session closed /
+ * detached Frame …) — none of which leave puppeteer with a meaningful
+ * post-navigation URL to recover.
+ * @param message - The raw error message from
+ *   `BrowserScrapeResult.error.message` (or any string that may carry
+ *   the sentinel inside a wrapper). `null` / `undefined` returns `false`.
+ * @returns `true` iff the message carries the `Page.goto returned null`
+ *   sentinel.
+ * @example
+ * ```ts
+ * isJsRedirectErrorShape('The method Page.goto returned null'); // → true
+ * isJsRedirectErrorShape('Navigation timeout of 60000 ms exceeded'); // → false
+ * isJsRedirectErrorShape(undefined); // → false
+ * ```
+ */
+export function isJsRedirectErrorShape(message) {
+    if (typeof message !== 'string' || message === '') {
+        return false;
+    }
+    return message.toLowerCase().includes(PAGE_GOTO_NULL_MARKER.toLowerCase());
+}

package/lib/crawler/is-puppeteer-fallback-candidate.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Decide whether a failed HEAD/GET pre-flight error message warrants one
+ * puppeteer fallback attempt. Pure and deterministic — the same message
+ * always gives the same answer, so the decision can be unit-tested without
+ * spinning up a browser.
+ * @param message - The pre-flight error message (typically the last rejected
+ *   `retryCall` attempt).
+ * @returns `true` when puppeteer should be tried once, `false` to give up.
+ * @example
+ * ```ts
+ * isPuppeteerFallbackCandidate('Timeout: https://slow.example.org/'); // true
+ * isPuppeteerFallbackCandidate('getaddrinfo ENOTFOUND host.invalid'); // false
+ * isPuppeteerFallbackCandidate('net::ERR_CERT_DATE_INVALID');         // false
+ * ```
+ */
+export declare function isPuppeteerFallbackCandidate(message: string): boolean;

package/lib/crawler/is-puppeteer-fallback-candidate.js

@@ -0,0 +1,63 @@
+import { classifyErrorKind } from '../classify-error-kind.js';
+/**
+ * Error kinds where a full puppeteer navigation has a realistic chance of
+ * succeeding even though the HEAD pre-flight (and its GET fallback) failed.
+ *
+ * These are the failure modes that a misconfigured WAF / middlebox / slow
+ * origin tends to produce against a bare HEAD/GET probe while still letting
+ * a real browser through — the browser uses a different request shape (full
+ * navigation lifecycle, JS-capable Accept headers, real cookies, optionally
+ * client TLS hints), and some hostile middleboxes only inspect the cheap
+ * shape. The fallback is one attempt only; if puppeteer also fails the URL
+ * is recorded as `status = -1` like before.
+ *
+ * Excluded kinds:
+ * - **dns / dns-transient** — DNS resolution happens at the OS level before
+ *   any browser request; puppeteer hits the same `getaddrinfo` outcome.
+ * - **tls** — Chromium will refuse the same certificate the Node TLS stack
+ *   refused (expired, wrong SAN, untrusted CA).
+ * - **client-blocked** — by definition the browser is the one rejecting.
+ * - **connection-refused** — TCP RST from the listener; same answer regardless
+ *   of client.
+ * - **connection-timeout** — `ETIMEDOUT` at the TCP connect stage means the
+ *   packets never reached the host (no SYN-ACK); puppeteer issues the same
+ *   `connect()` call and gets the same answer. Reserved for the middlebox
+ *   case (request reached the server, response timed out), which classifies
+ *   as `timeout` via the `NetTimeoutError "Timeout: <url>"` shape.
+ * - **local-network** — operator-side connectivity loss; nothing on this
+ *   machine will reach the host.
+ * - **protocol** — puppeteer lifecycle race; bouncing back to puppeteer
+ *   reproduces the same race.
+ * - **unknown** — by design. Spinning up a fresh Chromium for every
+ *   unclassifiable error is too expensive; if a real-world WAF / middlebox
+ *   pattern lands in `unknown`, add a matcher to {@link classifyErrorKind}
+ *   so it lands in one of the four included kinds above (where the fallback
+ *   has a meaningful chance of succeeding) instead of widening this set.
+ *
+ * `PreloadShortCircuitError`'s synthesised `getaddrinfo ENOTFOUND` message
+ * classifies into `dns` and is therefore filtered out automatically — no
+ * separate instanceof guard is needed at the call site.
+ */
+const PUPPETEER_FALLBACK_KINDS = new Set([
+    'timeout',
+    'connection-reset',
+    'parse-error',
+]);
+/**
+ * Decide whether a failed HEAD/GET pre-flight error message warrants one
+ * puppeteer fallback attempt. Pure and deterministic — the same message
+ * always gives the same answer, so the decision can be unit-tested without
+ * spinning up a browser.
+ * @param message - The pre-flight error message (typically the last rejected
+ *   `retryCall` attempt).
+ * @returns `true` when puppeteer should be tried once, `false` to give up.
+ * @example
+ * ```ts
+ * isPuppeteerFallbackCandidate('Timeout: https://slow.example.org/'); // true
+ * isPuppeteerFallbackCandidate('getaddrinfo ENOTFOUND host.invalid'); // false
+ * isPuppeteerFallbackCandidate('net::ERR_CERT_DATE_INVALID');         // false
+ * ```
+ */
+export function isPuppeteerFallbackCandidate(message) {
+    return PUPPETEER_FALLBACK_KINDS.has(classifyErrorKind(message));
+}

package/lib/crawler/link-list.d.ts

@@ -45,12 +45,32 @@ export default class LinkList {
      * @param resource.page - The scraped page data, if the scrape succeeded.
      * @param resource.error - The error object, if the scrape failed.
      * @param options - URL parsing options (e.g., `disableQueries`).
+     * @param completion - Behaviour overrides for how the redirect chain is folded into
+     *   the done-set.
+     * @param completion.includeRedirectPaths - When `false`, do NOT mark the URLs in
+     *   `resource.page.redirectPaths` as done. The default `true` preserves the
+     *   long-standing behaviour where a redirect chain (`/a → /b → /c`) folds every
+     *   intermediate URL into the done-set in a single sweep — correct for HTTP-layer
+     *   chains because the browser actually followed each hop, so reaching `/b` later
+     *   is a no-op. Pass `false` from the JS-redirect rescue: there `redirectPaths`
+     *   contains a single URL (the JS target Chromium navigated to after `page.goto()`
+     *   returned null), which the browser has NOT yet rendered. Folding it into the
+     *   done-set would make a subsequent `linkList.add(destinationUrl)` no-op (the
+     *   add() guard at line 51-53 refuses keys already in `#done`), so the dest URL
+     *   never enters `#pending` and the dealer never sees a push to enqueue. The
+     *   dealer's own `seen` Set in `#runDeal` (the gate `onPush` consults) is a
+     *   separate registry from `#done` — they are NOT kept in sync — but here it
+     *   does not matter: the rescue's `add()` is what feeds the eventual `enqueue()`
+     *   call, so blocking `add()` alone is enough to silently lose the JS target
+     *   from the archive.
      * @returns The constructed {@link Link} object, or `null` if the URL was not in the queue.
      */
     done(url: ExURL, scope: ReadonlyMap<string, readonly ExURL[]>, resource: {
         page?: PageData;
         error?: Error;
-    }, options: ParseURLOptions): Link | null;
+    }, options: ParseURLOptions, completion?: {
+        includeRedirectPaths?: boolean;
+    }): Link | null;
     /**
      * Get the current pending and in-progress URL lists.
      * @returns An object containing arrays of pending and in-progress URL strings.

package/lib/crawler/link-list.js

@@ -64,9 +64,27 @@ export default class LinkList {
      * @param resource.page - The scraped page data, if the scrape succeeded.
      * @param resource.error - The error object, if the scrape failed.
      * @param options - URL parsing options (e.g., `disableQueries`).
+     * @param completion - Behaviour overrides for how the redirect chain is folded into
+     *   the done-set.
+     * @param completion.includeRedirectPaths - When `false`, do NOT mark the URLs in
+     *   `resource.page.redirectPaths` as done. The default `true` preserves the
+     *   long-standing behaviour where a redirect chain (`/a → /b → /c`) folds every
+     *   intermediate URL into the done-set in a single sweep — correct for HTTP-layer
+     *   chains because the browser actually followed each hop, so reaching `/b` later
+     *   is a no-op. Pass `false` from the JS-redirect rescue: there `redirectPaths`
+     *   contains a single URL (the JS target Chromium navigated to after `page.goto()`
+     *   returned null), which the browser has NOT yet rendered. Folding it into the
+     *   done-set would make a subsequent `linkList.add(destinationUrl)` no-op (the
+     *   add() guard at line 51-53 refuses keys already in `#done`), so the dest URL
+     *   never enters `#pending` and the dealer never sees a push to enqueue. The
+     *   dealer's own `seen` Set in `#runDeal` (the gate `onPush` consults) is a
+     *   separate registry from `#done` — they are NOT kept in sync — but here it
+     *   does not matter: the rescue's `add()` is what feeds the eventual `enqueue()`
+     *   call, so blocking `add()` alone is enough to silently lose the JS target
+     *   from the archive.
      * @returns The constructed {@link Link} object, or `null` if the URL was not in the queue.
      */
-    done(url, scope, resource, options) {
+    done(url, scope, resource, options, completion) {
         const key = protocolAgnosticKey(url.withoutHashAndAuth);
         if (!(this.#pending.has(key) || this.#progress.has(key))) {
             return null;
@@ -96,8 +114,10 @@ export default class LinkList {
                 responseHeaders: resource.page.responseHeaders,
                 title: resource.page.meta.title,
             };
-            for (const path of resource.page.redirectPaths) {
-                urlList.add(protocolAgnosticKey(path));
+            if (completion?.includeRedirectPaths !== false) {
+                for (const path of resource.page.redirectPaths) {
+                    urlList.add(protocolAgnosticKey(path));
+                }
             }
         }
         if (resource.error?.message.includes('ERR_NAME_NOT_RESOLVED')) {

package/lib/crawler/normalize-content-type.d.ts

@@ -7,7 +7,7 @@
  * normalization, so `Text/HTML` or `text/html ` can otherwise reach the
  * database. Storing the canonical (trimmed, lower-cased) form lets the exact
  * SQL page-ness predicate (`WHERE contentType = 'text/html'`) agree with the
- * code-level {@link isHtmlContentType} check, which trims and lower-cases.
+ * code-level `isHtmlContentType` check, which trims and lower-cases.
  * @param contentType - The raw media type, or `null` when unknown.
  * @returns The trimmed, lower-cased media type, or `null` when unknown/blank.
  */

package/lib/crawler/normalize-content-type.js

@@ -7,7 +7,7 @@
  * normalization, so `Text/HTML` or `text/html ` can otherwise reach the
  * database. Storing the canonical (trimmed, lower-cased) form lets the exact
  * SQL page-ness predicate (`WHERE contentType = 'text/html'`) agree with the
- * code-level {@link isHtmlContentType} check, which trims and lower-cases.
+ * code-level `isHtmlContentType` check, which trims and lower-cases.
  * @param contentType - The raw media type, or `null` when unknown.
  * @returns The trimmed, lower-cased media type, or `null` when unknown/blank.
  */

package/lib/crawler/plan-sub-resource-emits.d.ts

@@ -0,0 +1,63 @@
+import type { CrawlerEventTypes } from './types.js';
+import type { PageSource } from '../archive/types.js';
+import type { ResourceEntry } from '@d-zero/beholder';
+/**
+ * Planned `response` emit produced by {@link planSubResourceEmits}.
+ */
+export interface PlannedResponseEmit {
+    /** The resource payload to attach to the `response` event. */
+    resource: CrawlerEventTypes['response']['resource'];
+    /**
+     * The `source` field propagated to the `response` event. Resolved from
+     * the parent page's lineage via {@link deriveResourceSource} — pinning
+     * this through a planning step (rather than computing it inline in
+     * `#handleResources`) lets the wire-up be unit-tested without spinning
+     * up the puppeteer mock stack.
+     */
+    source: PageSource | undefined;
+}
+/**
+ * Planned `responseReferrers` emit produced by {@link planSubResourceEmits}.
+ * Always emitted, regardless of whether the resource is new — `isNew` only
+ * gates the `response` event.
+ */
+export interface PlannedReferrerEmit {
+    /** The page URL that triggered the sub-resource fetch. */
+    url: string;
+    /** The resource URL (hash stripped to match the storage key). */
+    src: string;
+}
+/**
+ * Output of {@link planSubResourceEmits}: the deduped `response` plan and
+ * the per-resource `responseReferrers` plan, side-by-side.
+ */
+export interface SubResourceEmitPlan {
+    /** `response` events to emit (new resources only). */
+    responseEmits: PlannedResponseEmit[];
+    /** `responseReferrers` events to emit (every resource, even seen ones). */
+    referrerEmits: PlannedReferrerEmit[];
+}
+/**
+ * Decide which sub-resource `response` / `responseReferrers` events the
+ * crawler should emit for a page render, with the parent's source lineage
+ * baked into every `response` event's `source` field.
+ *
+ * Pure function — takes the resources captured during the render plus the
+ * seen-resource set and the parent's source, returns the emit plan. The
+ * caller (`Crawler.#handleResources`) is responsible for iterating the
+ * plan through its event emitter. Splitting "decide what to emit" from
+ * "actually emit" is what makes the lineage propagation contract
+ * unit-testable: the previous shape inlined `emit('response', { ...
+ * source: deriveResourceSource(...) })` and could only be exercised via a
+ * full scrape with a mocked puppeteer stack, which left the `source`
+ * value half of the contract effectively un-pinned.
+ *
+ * Mutates `seenResources` as a side effect — every captured resource is
+ * recorded as seen so the next call dedupes correctly. This mirrors the
+ * `Crawler.#resources` Set semantics that the planner is designed to share.
+ * @param resources - Sub-resource entries captured during the page render.
+ * @param parentSource - Merged source of the page being rendered, as resolved by `Crawler.#resolveParentSource`.
+ * @param seenResources - Mutable set of already-seen resource keys (mutated in place).
+ * @returns The plan of `response` + `responseReferrers` emits to dispatch.
+ */
+export declare function planSubResourceEmits(resources: ResourceEntry[], parentSource: PageSource | undefined, seenResources: Set<string>): SubResourceEmitPlan;

package/lib/crawler/plan-sub-resource-emits.js

@@ -0,0 +1,44 @@
+import { deriveResourceSource } from './derive-resource-source.js';
+import { handleResourceResponse } from './handle-resource-response.js';
+/**
+ * Decide which sub-resource `response` / `responseReferrers` events the
+ * crawler should emit for a page render, with the parent's source lineage
+ * baked into every `response` event's `source` field.
+ *
+ * Pure function — takes the resources captured during the render plus the
+ * seen-resource set and the parent's source, returns the emit plan. The
+ * caller (`Crawler.#handleResources`) is responsible for iterating the
+ * plan through its event emitter. Splitting "decide what to emit" from
+ * "actually emit" is what makes the lineage propagation contract
+ * unit-testable: the previous shape inlined `emit('response', { ...
+ * source: deriveResourceSource(...) })` and could only be exercised via a
+ * full scrape with a mocked puppeteer stack, which left the `source`
+ * value half of the contract effectively un-pinned.
+ *
+ * Mutates `seenResources` as a side effect — every captured resource is
+ * recorded as seen so the next call dedupes correctly. This mirrors the
+ * `Crawler.#resources` Set semantics that the planner is designed to share.
+ * @param resources - Sub-resource entries captured during the page render.
+ * @param parentSource - Merged source of the page being rendered, as resolved by `Crawler.#resolveParentSource`.
+ * @param seenResources - Mutable set of already-seen resource keys (mutated in place).
+ * @returns The plan of `response` + `responseReferrers` emits to dispatch.
+ */
+export function planSubResourceEmits(resources, parentSource, seenResources) {
+    const subResourceSource = deriveResourceSource(parentSource);
+    const responseEmits = [];
+    const referrerEmits = [];
+    for (const { resource, pageUrl } of resources) {
+        const { isNew } = handleResourceResponse(resource, seenResources);
+        if (isNew) {
+            responseEmits.push({
+                resource: resource,
+                source: subResourceSource,
+            });
+        }
+        referrerEmits.push({
+            url: pageUrl,
+            src: resource.url.withoutHash,
+        });
+    }
+    return { responseEmits, referrerEmits };
+}

package/lib/crawler/preload-short-circuit-error.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Thrown by `Crawler.#sendHeadRequest` when the target URL's hostname is in
+ * `dnsBurnedHostCache` — both session-learned and preload-seeded burns
+ * land here.
+ *
+ * The orchestrator's `crawler.on('error', …)` handler tests `instanceof` and
+ * skips writing this error to `crawl_errors` / `error.log`, so the same
+ * preload data isn't re-amplified on subsequent crawls. `pages.status = -1`
+ * still gets set through the normal scrape-error path.
+ *
+ * The message embeds the `ENOTFOUND` token so any downstream consumer that
+ * runs `classifyErrorKind` over it (e.g. dealer log forwarders) still gets
+ * the `'dns'` classification.
+ */
+export declare class PreloadShortCircuitError extends Error {
+    /** Sniffable flag for callers that prefer duck-typing over instanceof. */
+    readonly isPreloadShortCircuit: true;
+    /**
+     * @param host - The DNS-burned hostname (already lowercased / Punycoded).
+     */
+    constructor(host: string);
+}

package/lib/crawler/preload-short-circuit-error.js

@@ -0,0 +1,25 @@
+/**
+ * Thrown by `Crawler.#sendHeadRequest` when the target URL's hostname is in
+ * `dnsBurnedHostCache` — both session-learned and preload-seeded burns
+ * land here.
+ *
+ * The orchestrator's `crawler.on('error', …)` handler tests `instanceof` and
+ * skips writing this error to `crawl_errors` / `error.log`, so the same
+ * preload data isn't re-amplified on subsequent crawls. `pages.status = -1`
+ * still gets set through the normal scrape-error path.
+ *
+ * The message embeds the `ENOTFOUND` token so any downstream consumer that
+ * runs `classifyErrorKind` over it (e.g. dealer log forwarders) still gets
+ * the `'dns'` classification.
+ */
+export class PreloadShortCircuitError extends Error {
+    /** Sniffable flag for callers that prefer duck-typing over instanceof. */
+    isPreloadShortCircuit = true;
+    /**
+     * @param host - The DNS-burned hostname (already lowercased / Punycoded).
+     */
+    constructor(host) {
+        super(`getaddrinfo ENOTFOUND ${host}`);
+        this.name = 'PreloadShortCircuitError';
+    }
+}

package/lib/crawler/should-burn-host.d.ts

@@ -0,0 +1,78 @@
+import type { ErrorKind } from '../types.js';
+/**
+ * Inputs to {@link shouldBurnHost}.
+ */
+export interface ShouldBurnHostParams {
+    /**
+     * The {@link ErrorKind} classified from the final-attempt error message
+     * (i.e. the error that ended the retry loop in `Crawler.#sendHeadRequest`'s
+     * `onGiveUp`).
+     */
+    errorKind: ErrorKind;
+    /**
+     * Lower-cased hostname whose URL just exhausted retries. Must already be
+     * normalised by the caller — `dnsBurnedHostCache` keys are
+     * `url.hostname.toLowerCase()`, so this guard reuses that exact form to
+     * stay consistent across the two sites.
+     */
+    host: string;
+    /**
+     * Set of hostnames that have had at least one successful
+     * `fetchDestination` response in this session. `ReadonlySet` because the
+     * decision is read-only — populating the set is the caller's job.
+     */
+    successfulHosts: ReadonlySet<string>;
+}
+/**
+ * Burn the host iff the final-attempt kind is `'dns'` AND the host has no
+ * session-success record. Pure function — unit-testable in isolation from
+ * the `Crawler` instance, the in-memory caches, and the dealer's retry
+ * plumbing.
+ *
+ * **Why the session-success gate exists**: the first worker to exhaust
+ * retries with `getaddrinfo ENOTFOUND` would otherwise burn the host and
+ * make every subsequent URL on it short-circuit immediately via
+ * `PreloadShortCircuitError`, draining the dealer's work queue in seconds
+ * and collapsing the crawl into a degenerate `crawlEnd`. When the cause is
+ * a local-network blip (operator's WiFi → tethering / VPN flip / ISP DNS
+ * hiccup mid-crawl) rather than a dead domain, the host was demonstrably
+ * alive moments earlier — earlier successes on it are recorded in
+ * `successfulHosts`, so the cascade is suppressed.
+ *
+ * **Why `'dns-transient'` is excluded**: `EAI_AGAIN` / `EREFUSED` are
+ * absorbed by the retry layer within the session; a final-attempt
+ * `'dns-transient'` is rare enough that we'd rather pay the per-URL retry
+ * cost than wrongly fast-fail a healthy host that flapped briefly. Only the
+ * stronger `'dns'` kind is a candidate for burning.
+ *
+ * **Known limitation — first-URL false positives**: a host whose very first
+ * URL of the session hits a real network blip exhausts its retry budget
+ * before any URL has succeeded, so `successfulHosts` is still empty and the
+ * host IS burned. That URL's siblings on the same host then short-circuit.
+ * Acceptable trade-off: the alternative ("never burn anything") regresses
+ * the dead-domain fast-fail behavior that the burn cache exists for. The
+ * pause-dealer-on-outage layer (separate issue) covers this gap by
+ * detecting the outage BEFORE the first retry budget runs out.
+ *
+ * **Known limitation — preload-seeded burns are not un-burned**: a host
+ * added by `#preloadDnsBurnedHostCache` from the archive's `crawl_errors`
+ * trips `PreloadShortCircuitError` at the top of `#sendHeadRequest` and
+ * never reaches `fetchDestination`, so `successfulHosts` is never
+ * populated for it in this session. This is intentional — preload only
+ * seeds hosts whose archive evidence is "DNS-failed with no recovery", so
+ * un-burning them on a single transient success could re-introduce the
+ * cascade we are trying to prevent.
+ *
+ * **What does NOT contribute to `successfulHosts`**: external pages
+ * traversed via `fetchExternal: false` skip the HEAD pre-flight entirely
+ * (the crawler stamps a synthetic `PageData` without touching the
+ * network), so the host is not recorded as alive even if the same host
+ * appears in the crawled-internal scope. Callers must populate the set
+ * from real HTTP-response observations only.
+ * @param params - See {@link ShouldBurnHostParams}.
+ * @param params.errorKind
+ * @param params.host
+ * @param params.successfulHosts
+ * @returns `true` if the burn cache should record this host, `false` otherwise.
+ */
+export declare function shouldBurnHost({ errorKind, host, successfulHosts, }: ShouldBurnHostParams): boolean;

package/lib/crawler/should-burn-host.js

@@ -0,0 +1,61 @@
+/**
+ * Burn the host iff the final-attempt kind is `'dns'` AND the host has no
+ * session-success record. Pure function — unit-testable in isolation from
+ * the `Crawler` instance, the in-memory caches, and the dealer's retry
+ * plumbing.
+ *
+ * **Why the session-success gate exists**: the first worker to exhaust
+ * retries with `getaddrinfo ENOTFOUND` would otherwise burn the host and
+ * make every subsequent URL on it short-circuit immediately via
+ * `PreloadShortCircuitError`, draining the dealer's work queue in seconds
+ * and collapsing the crawl into a degenerate `crawlEnd`. When the cause is
+ * a local-network blip (operator's WiFi → tethering / VPN flip / ISP DNS
+ * hiccup mid-crawl) rather than a dead domain, the host was demonstrably
+ * alive moments earlier — earlier successes on it are recorded in
+ * `successfulHosts`, so the cascade is suppressed.
+ *
+ * **Why `'dns-transient'` is excluded**: `EAI_AGAIN` / `EREFUSED` are
+ * absorbed by the retry layer within the session; a final-attempt
+ * `'dns-transient'` is rare enough that we'd rather pay the per-URL retry
+ * cost than wrongly fast-fail a healthy host that flapped briefly. Only the
+ * stronger `'dns'` kind is a candidate for burning.
+ *
+ * **Known limitation — first-URL false positives**: a host whose very first
+ * URL of the session hits a real network blip exhausts its retry budget
+ * before any URL has succeeded, so `successfulHosts` is still empty and the
+ * host IS burned. That URL's siblings on the same host then short-circuit.
+ * Acceptable trade-off: the alternative ("never burn anything") regresses
+ * the dead-domain fast-fail behavior that the burn cache exists for. The
+ * pause-dealer-on-outage layer (separate issue) covers this gap by
+ * detecting the outage BEFORE the first retry budget runs out.
+ *
+ * **Known limitation — preload-seeded burns are not un-burned**: a host
+ * added by `#preloadDnsBurnedHostCache` from the archive's `crawl_errors`
+ * trips `PreloadShortCircuitError` at the top of `#sendHeadRequest` and
+ * never reaches `fetchDestination`, so `successfulHosts` is never
+ * populated for it in this session. This is intentional — preload only
+ * seeds hosts whose archive evidence is "DNS-failed with no recovery", so
+ * un-burning them on a single transient success could re-introduce the
+ * cascade we are trying to prevent.
+ *
+ * **What does NOT contribute to `successfulHosts`**: external pages
+ * traversed via `fetchExternal: false` skip the HEAD pre-flight entirely
+ * (the crawler stamps a synthetic `PageData` without touching the
+ * network), so the host is not recorded as alive even if the same host
+ * appears in the crawled-internal scope. Callers must populate the set
+ * from real HTTP-response observations only.
+ * @param params - See {@link ShouldBurnHostParams}.
+ * @param params.errorKind
+ * @param params.host
+ * @param params.successfulHosts
+ * @returns `true` if the burn cache should record this host, `false` otherwise.
+ */
+export function shouldBurnHost({ errorKind, host, successfulHosts, }) {
+    if (errorKind !== 'dns') {
+        return false;
+    }
+    if (successfulHosts.has(host)) {
+        return false;
+    }
+    return true;
+}

package/lib/crawler/should-get-fallback-on-head-failure.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Decide whether a HEAD pre-flight failure should trigger a GET retry and
+ * stay OUT of `destinationCache`.
+ *
+ * The two contracts are intentionally tied to one helper: anything we'll GET
+ * later because the HEAD answer might be wrong is also the same thing we
+ * must NOT freeze into the per-session cache (or the second attempt would
+ * hit the stale cached failure and skip the live retry that
+ * `Crawler.#sendHeadRequest`'s `HEAD_TIMEOUT_ESCALATION_MS` is supposed to
+ * pay for).
+ *
+ * The eligible kinds are:
+ *
+ * - **`NetTimeoutError`** — the HEAD pre-flight race fired without a
+ *   server response. The escalating retry can still succeed against a
+ *   slow-but-reachable host.
+ * - **`parse-error`** — `Parse Error` / `Expected HTTP/` / `Unexpected end
+ *   of stream`. Usually a middlebox rewriting / truncating the HEAD reply
+ *   the GET path traverses differently.
+ * - **`connection-reset`** — `ECONNRESET` / `ERR_CONNECTION_RESET` etc.
+ *   middlebox dropping the connection mid-response; a retry frequently
+ *   succeeds.
+ *
+ * Everything else — DNS, TLS, refused, blocked, plain timeout — is treated
+ * as a persistent within-session verdict and IS cached so repeated calls on
+ * the same host pay the network cost once.
+ * @param error - The `Error` raised by the HEAD attempt.
+ * @returns `true` when the error warrants a GET fallback AND a cache skip.
+ * @example
+ * ```ts
+ * shouldGetFallbackOnHeadFailure(new NetTimeoutError(url)); // true
+ * shouldGetFallbackOnHeadFailure(new Error('read ECONNRESET')); // true
+ * shouldGetFallbackOnHeadFailure(new Error('Parse Error')); // true
+ * shouldGetFallbackOnHeadFailure(new Error('getaddrinfo ENOTFOUND host')); // false
+ * shouldGetFallbackOnHeadFailure(new Error('ERR_CERT_DATE_INVALID')); // false
+ * ```
+ */
+export declare function shouldGetFallbackOnHeadFailure(error: Error): boolean;