@nitpicker/crawler 0.11.0 → 0.12.0

package/lib/crawler/derive-js-redirect-target.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Decide whether a puppeteer post-navigation URL represents a real
+ * client-side redirect (JS / meta-refresh) or just noise.
+ *
+ * **Why this helper exists:** when `page.goto()` returns `null`, the upstream
+ * scraper throws `The method Page.goto returned null` — classified as
+ * `protocol`, neither permanent nor a puppeteer-fallback kind, so
+ * `--retry-failed` never converges. Reading `page.url()` after the throw is
+ * the only way to recover the destination Chromium actually navigated to (the
+ * thrown error carries no URL), so we can record the source as a redirect
+ * edge instead of a hard `status = -1`. But `page.url()` can also report
+ * uninformative values — `about:blank` before the first navigation completes,
+ * the original URL when nothing happened — and a naive "different ?" check
+ * would create a phantom redirect every time the navigation simply failed at
+ * the same URL.
+ *
+ * The filter is intentionally narrow: anything that does not look like an
+ * `http(s):` URL semantically distinct from the originally requested
+ * location is discarded. Edge cases the test pins:
+ *
+ * - identity after URL canonicalisation (case / trailing-slash / default
+ *   port / credentials / fragment) → null. WHATWG URL parsing handles
+ *   `https://Example.COM` vs `https://example.com/`, `https://host:443/`
+ *   vs `https://host/`, `https://u:p@host/x` vs `https://host/x`, etc.
+ * - `about:blank` / `chrome-error://...` / `data:` / `file:` → null
+ *   (browser-internal sentinels, never a legitimate destination)
+ * - empty / whitespace string → null (defensive against
+ *   `page.url()` returning `''` on a brand-new context)
+ * - any genuine `http(s):` URL semantically different from the source →
+ *   returned credential-/fragment-stripped (defence-in-depth against a
+ *   pre-RFC server issuing `Location: https://user:pass@host/path`, which
+ *   would otherwise persist credentials into the `.nitpicker` archive
+ *   — exactly the leak class the scope-auth-leak guard at the navigation
+ *   side blocks)
+ *
+ * The caller does NOT need to pre-normalise URLs — this helper normalises
+ * both sides via the WHATWG URL parser before comparing. The production
+ * call site passes `url.withoutHashAndAuth` for `originalUrl`, but the
+ * normalisation here is idempotent so passing `url.href` would also work.
+ * @param originalUrl - The URL puppeteer was asked to navigate to.
+ * @param postNavigationUrl - The URL reported by `page.url()` after the
+ *   throw. May be `null` / `undefined` when reading the URL itself failed.
+ * @returns The credential-/fragment-stripped post-navigation URL when it
+ *   represents a real JS redirect, otherwise `null`.
+ * @example
+ * ```ts
+ * deriveJsRedirectTarget(
+ *   'https://www.example.com/old',
+ *   'https://www.example.com/new',
+ * ); // → 'https://www.example.com/new'
+ *
+ * deriveJsRedirectTarget('https://www.example.com/old', 'about:blank');
+ * // → null
+ *
+ * // Case / trailing-slash noise — no phantom self-redirect:
+ * deriveJsRedirectTarget(
+ *   'https://www.example.com',
+ *   'https://www.example.com/',
+ * ); // → null
+ *
+ * // Credentials in destination — stripped before return:
+ * deriveJsRedirectTarget(
+ *   'https://www.example.com/',
+ *   'https://user:pass@www.example.com/dest',
+ * ); // → 'https://www.example.com/dest'
+ * ```
+ */
+export declare function deriveJsRedirectTarget(originalUrl: string, postNavigationUrl: string | null | undefined): string | null;

package/lib/crawler/derive-js-redirect-target.js

@@ -0,0 +1,129 @@
+/** Matches only the `http:` and `https:` schemes (case-insensitive). */
+const HTTP_SCHEME_PATTERN = /^https?:\/\//i;
+/**
+ * Canonicalise a URL for the JS-redirect identity check and return value:
+ * strip credentials and fragment, normalise case / default-ports /
+ * trailing-slash via the WHATWG URL parser. Returns `null` when the input
+ * is unparseable.
+ *
+ * Used for both sides of the comparison so case-only or trailing-slash-only
+ * differences (`https://Example.com` vs `https://example.com/`) do not
+ * produce phantom self-redirects, and for the returned target so a pre-RFC
+ * `Location: https://user:pass@host/path` is never persisted into the
+ * archive with credentials (same threat model the scope-auth-leak guard
+ * mitigates on the navigation side).
+ * @param rawUrl - The URL string to canonicalise.
+ * @returns The canonical URL string, or `null` when unparseable.
+ */
+function canonicaliseForComparison(rawUrl) {
+    try {
+        const parsed = new URL(rawUrl);
+        parsed.username = '';
+        parsed.password = '';
+        parsed.hash = '';
+        return parsed.href;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Decide whether a puppeteer post-navigation URL represents a real
+ * client-side redirect (JS / meta-refresh) or just noise.
+ *
+ * **Why this helper exists:** when `page.goto()` returns `null`, the upstream
+ * scraper throws `The method Page.goto returned null` — classified as
+ * `protocol`, neither permanent nor a puppeteer-fallback kind, so
+ * `--retry-failed` never converges. Reading `page.url()` after the throw is
+ * the only way to recover the destination Chromium actually navigated to (the
+ * thrown error carries no URL), so we can record the source as a redirect
+ * edge instead of a hard `status = -1`. But `page.url()` can also report
+ * uninformative values — `about:blank` before the first navigation completes,
+ * the original URL when nothing happened — and a naive "different ?" check
+ * would create a phantom redirect every time the navigation simply failed at
+ * the same URL.
+ *
+ * The filter is intentionally narrow: anything that does not look like an
+ * `http(s):` URL semantically distinct from the originally requested
+ * location is discarded. Edge cases the test pins:
+ *
+ * - identity after URL canonicalisation (case / trailing-slash / default
+ *   port / credentials / fragment) → null. WHATWG URL parsing handles
+ *   `https://Example.COM` vs `https://example.com/`, `https://host:443/`
+ *   vs `https://host/`, `https://u:p@host/x` vs `https://host/x`, etc.
+ * - `about:blank` / `chrome-error://...` / `data:` / `file:` → null
+ *   (browser-internal sentinels, never a legitimate destination)
+ * - empty / whitespace string → null (defensive against
+ *   `page.url()` returning `''` on a brand-new context)
+ * - any genuine `http(s):` URL semantically different from the source →
+ *   returned credential-/fragment-stripped (defence-in-depth against a
+ *   pre-RFC server issuing `Location: https://user:pass@host/path`, which
+ *   would otherwise persist credentials into the `.nitpicker` archive
+ *   — exactly the leak class the scope-auth-leak guard at the navigation
+ *   side blocks)
+ *
+ * The caller does NOT need to pre-normalise URLs — this helper normalises
+ * both sides via the WHATWG URL parser before comparing. The production
+ * call site passes `url.withoutHashAndAuth` for `originalUrl`, but the
+ * normalisation here is idempotent so passing `url.href` would also work.
+ * @param originalUrl - The URL puppeteer was asked to navigate to.
+ * @param postNavigationUrl - The URL reported by `page.url()` after the
+ *   throw. May be `null` / `undefined` when reading the URL itself failed.
+ * @returns The credential-/fragment-stripped post-navigation URL when it
+ *   represents a real JS redirect, otherwise `null`.
+ * @example
+ * ```ts
+ * deriveJsRedirectTarget(
+ *   'https://www.example.com/old',
+ *   'https://www.example.com/new',
+ * ); // → 'https://www.example.com/new'
+ *
+ * deriveJsRedirectTarget('https://www.example.com/old', 'about:blank');
+ * // → null
+ *
+ * // Case / trailing-slash noise — no phantom self-redirect:
+ * deriveJsRedirectTarget(
+ *   'https://www.example.com',
+ *   'https://www.example.com/',
+ * ); // → null
+ *
+ * // Credentials in destination — stripped before return:
+ * deriveJsRedirectTarget(
+ *   'https://www.example.com/',
+ *   'https://user:pass@www.example.com/dest',
+ * ); // → 'https://www.example.com/dest'
+ * ```
+ */
+export function deriveJsRedirectTarget(originalUrl, postNavigationUrl) {
+    if (typeof postNavigationUrl !== 'string') {
+        return null;
+    }
+    const trimmed = postNavigationUrl.trim();
+    if (trimmed === '') {
+        return null;
+    }
+    // Browser-internal sentinels: `about:blank` appears before the first
+    // navigation completes, `chrome-error://...` after a network error
+    // renders the Chromium error page, and `data:` / `file:` / `javascript:`
+    // can never be the destination of an off-page redirect we want to record.
+    // `http:` / `https:` is the only safe positive match.
+    if (!HTTP_SCHEME_PATTERN.test(trimmed)) {
+        return null;
+    }
+    const destinationCanonical = canonicaliseForComparison(trimmed);
+    if (destinationCanonical === null) {
+        return null;
+    }
+    const originalCanonical = canonicaliseForComparison(originalUrl);
+    // `originalCanonical === null` only happens for a degenerate (unparseable)
+    // `originalUrl` — fall back to a raw string check so we never claim a
+    // redirect we cannot prove. The production call site always supplies a
+    // parseable form.
+    if (originalCanonical === null) {
+        return trimmed === originalUrl ? null : destinationCanonical;
+    }
+    if (destinationCanonical === originalCanonical) {
+        return null;
+    }
+    return destinationCanonical;
+}

package/lib/crawler/derive-resource-source.d.ts

@@ -1,23 +1,33 @@
-import type { InventoryMode } from './types.js';
 import type { PageSource } from '../archive/types.js';
 /**
  * Decide which {@link PageSource} label a newly-captured sub-resource row
- * (CSS / JS / image / fetch response) should carry.
+ * (CSS / JS / image / fetch response) should carry, based on the lineage of
+ * the page that is currently being rendered.
  *
  * Sub-resources are NEVER themselves seeds — even when puppeteer is
- * rendering a page that *is* an inventory seed, the assets it pulls in
- * are downstream and must be labelled `'inventory-discovered'`. The seed
- * label is reserved for URLs that were explicitly handed in by the user
- * via the `--inventory` file.
+ * rendering a page that *is* an inventory seed, the assets it pulls in are
+ * downstream and must be labelled `'inventory-discovered'`. The seed label
+ * is reserved for URLs that were explicitly handed in by the user via the
+ * `--inventory` file. Likewise, when a sub-resource is captured during a
+ * render of an `'inventory-discovered'` page (a page reached transitively
+ * through the inventory chain), the asset is still inventory-discovered —
+ * not a new seed.
  *
- * Outside inventory mode (`inventoryMode === null`) this returns
- * `undefined` so the caller emits no `source` and the DB DEFAULT
- * (`'crawled'`) lands on the row — keeps the normal crawl path
- * untouched. This is the sub-resource counterpart of
- * {@link import('./derive-page-source.js').derivePageSource}; the two
- * helpers exist as a pair so a future addition to {@link PageSource}
- * forces a parallel update.
- * @param inventoryMode - Inventory-mode config from `CrawlerOptions.inventoryMode`, or `null` outside `--inventory`.
+ * Outside the inventory chain (parent is `'crawled'` or has no source
+ * record) this returns `undefined` so the caller emits no `source` and the
+ * DB DEFAULT (`'crawled'`) lands on the row, leaving the normal crawl path
+ * untouched.
+ *
+ * `parentSource` must reflect the MERGED source of the page being scraped:
+ * for an active `--inventory` session that comes from
+ * {@link import('./derive-page-source.js').derivePageSource}, for a
+ * `--resume` / `--retry-failed` session it comes from a DB lookup
+ * (`PageSourceLookup`) because `inventoryMode` is not persisted across
+ * sessions. The two-stage resolution is what keeps sub-resource labels
+ * correct on resume: even though the orchestrator no longer carries the
+ * inventory seed set in memory, the DB still records the parent's lineage
+ * and we propagate it forward.
+ * @param parentSource - Merged source of the page that is producing this sub-resource (or `undefined` for an unknown / `'crawled'` parent).
  * @returns The label to write to `resources.source`, or `undefined` for the DB default.
  */
-export declare function deriveResourceSource(inventoryMode: InventoryMode | null): PageSource | undefined;
+export declare function deriveResourceSource(parentSource: PageSource | undefined): PageSource | undefined;

package/lib/crawler/derive-resource-source.js

@@ -1,26 +1,37 @@
 /**
  * Decide which {@link PageSource} label a newly-captured sub-resource row
- * (CSS / JS / image / fetch response) should carry.
+ * (CSS / JS / image / fetch response) should carry, based on the lineage of
+ * the page that is currently being rendered.
  *
  * Sub-resources are NEVER themselves seeds — even when puppeteer is
- * rendering a page that *is* an inventory seed, the assets it pulls in
- * are downstream and must be labelled `'inventory-discovered'`. The seed
- * label is reserved for URLs that were explicitly handed in by the user
- * via the `--inventory` file.
+ * rendering a page that *is* an inventory seed, the assets it pulls in are
+ * downstream and must be labelled `'inventory-discovered'`. The seed label
+ * is reserved for URLs that were explicitly handed in by the user via the
+ * `--inventory` file. Likewise, when a sub-resource is captured during a
+ * render of an `'inventory-discovered'` page (a page reached transitively
+ * through the inventory chain), the asset is still inventory-discovered —
+ * not a new seed.
  *
- * Outside inventory mode (`inventoryMode === null`) this returns
- * `undefined` so the caller emits no `source` and the DB DEFAULT
- * (`'crawled'`) lands on the row — keeps the normal crawl path
- * untouched. This is the sub-resource counterpart of
- * {@link import('./derive-page-source.js').derivePageSource}; the two
- * helpers exist as a pair so a future addition to {@link PageSource}
- * forces a parallel update.
- * @param inventoryMode - Inventory-mode config from `CrawlerOptions.inventoryMode`, or `null` outside `--inventory`.
+ * Outside the inventory chain (parent is `'crawled'` or has no source
+ * record) this returns `undefined` so the caller emits no `source` and the
+ * DB DEFAULT (`'crawled'`) lands on the row, leaving the normal crawl path
+ * untouched.
+ *
+ * `parentSource` must reflect the MERGED source of the page being scraped:
+ * for an active `--inventory` session that comes from
+ * {@link import('./derive-page-source.js').derivePageSource}, for a
+ * `--resume` / `--retry-failed` session it comes from a DB lookup
+ * (`PageSourceLookup`) because `inventoryMode` is not persisted across
+ * sessions. The two-stage resolution is what keeps sub-resource labels
+ * correct on resume: even though the orchestrator no longer carries the
+ * inventory seed set in memory, the DB still records the parent's lineage
+ * and we propagate it forward.
+ * @param parentSource - Merged source of the page that is producing this sub-resource (or `undefined` for an unknown / `'crawled'` parent).
  * @returns The label to write to `resources.source`, or `undefined` for the DB default.
  */
-export function deriveResourceSource(inventoryMode) {
-    if (inventoryMode === null) {
-        return undefined;
+export function deriveResourceSource(parentSource) {
+    if (parentSource === 'inventory-seed' || parentSource === 'inventory-discovered') {
+        return 'inventory-discovered';
     }
-    return 'inventory-discovered';
+    return undefined;
 }

package/lib/crawler/dns-burned-host-cache.d.ts

@@ -0,0 +1,26 @@
+import type { ErrorKind } from '../types.js';
+/**
+ * In-memory set of hostnames known to be unreachable due to DNS errors
+ * (e.g. `getaddrinfo ENOTFOUND`).
+ *
+ * Lives for one crawl session — cleared by {@link clearDnsBurnedHostCache} at
+ * the same four orchestrator sites as {@link destinationCache}. Hosts are
+ * marked in two ways:
+ *
+ * - **Session learning**: the `onGiveUp` callback of `#sendHeadRequest`
+ *   classifies the final retry error and stores `'dns'` when the matcher
+ *   fires. The next URL on the same hostname short-circuits before retry.
+ * - **Session preload**: re-open paths (`append` / `inventory` / `retryFailed`
+ *   / `resume`) call {@link Archive.listDnsBurnedHostCandidates} and seed the
+ *   map from `crawl_errors`, so previously-burned hosts cost zero retries on
+ *   the next crawl.
+ *
+ * Keys are always `url.hostname.toLowerCase()` — WHATWG URL has already
+ * Punycoded IDNs and stripped the port, so no extra normalization is needed
+ * for IPv4 / IPv6 literals or international hostnames.
+ *
+ * The value records the originating {@link ErrorKind}. Only `'dns'` is set
+ * today; the union shape is preserved so future error classes
+ * (e.g. `tls`, `connection-refused`) can extend the same cache.
+ */
+export declare const dnsBurnedHostCache: Map<string, ErrorKind>;

package/lib/crawler/dns-burned-host-cache.js

@@ -0,0 +1,25 @@
+/**
+ * In-memory set of hostnames known to be unreachable due to DNS errors
+ * (e.g. `getaddrinfo ENOTFOUND`).
+ *
+ * Lives for one crawl session — cleared by {@link clearDnsBurnedHostCache} at
+ * the same four orchestrator sites as {@link destinationCache}. Hosts are
+ * marked in two ways:
+ *
+ * - **Session learning**: the `onGiveUp` callback of `#sendHeadRequest`
+ *   classifies the final retry error and stores `'dns'` when the matcher
+ *   fires. The next URL on the same hostname short-circuits before retry.
+ * - **Session preload**: re-open paths (`append` / `inventory` / `retryFailed`
+ *   / `resume`) call {@link Archive.listDnsBurnedHostCandidates} and seed the
+ *   map from `crawl_errors`, so previously-burned hosts cost zero retries on
+ *   the next crawl.
+ *
+ * Keys are always `url.hostname.toLowerCase()` — WHATWG URL has already
+ * Punycoded IDNs and stripped the port, so no extra normalization is needed
+ * for IPv4 / IPv6 literals or international hostnames.
+ *
+ * The value records the originating {@link ErrorKind}. Only `'dns'` is set
+ * today; the union shape is preserved so future error classes
+ * (e.g. `tls`, `connection-refused`) can extend the same cache.
+ */
+export const dnsBurnedHostCache = new Map();

package/lib/crawler/dns-burned-host-short-circuit-counter.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Session-scoped counter for how many URL fetches were short-circuited by the
+ * {@link dnsBurnedHostCache}. The orchestrator reads this at crawl shutdown
+ * to emit a `[preload] Short-circuited N URL(s)` summary line, then
+ * {@link clearDnsBurnedHostCache} zeroes it for the next session.
+ *
+ * Exposed as a mutable object (not a plain `let`) so that the counter remains
+ * a single shared reference across crawler / orchestrator imports — `let`
+ * bindings cannot be mutated from another module.
+ */
+export declare const dnsBurnedHostShortCircuitCounter: {
+    count: number;
+};

package/lib/crawler/dns-burned-host-short-circuit-counter.js

@@ -0,0 +1,11 @@
+/**
+ * Session-scoped counter for how many URL fetches were short-circuited by the
+ * {@link dnsBurnedHostCache}. The orchestrator reads this at crawl shutdown
+ * to emit a `[preload] Short-circuited N URL(s)` summary line, then
+ * {@link clearDnsBurnedHostCache} zeroes it for the next session.
+ *
+ * Exposed as a mutable object (not a plain `let`) so that the counter remains
+ * a single shared reference across crawler / orchestrator imports — `let`
+ * bindings cannot be mutated from another module.
+ */
+export const dnsBurnedHostShortCircuitCounter = { count: 0 };

package/lib/crawler/fetch-destination.d.ts

@@ -20,19 +20,27 @@ export interface FetchDestinationParams {
     };
     /** User-Agent string to send with the request. */
     readonly userAgent?: string;
+    /**
+     * Race timeout for the network request in milliseconds. Defaults to
+     * {@link DEFAULT_HEAD_TIMEOUT_MS} (10s). `Crawler.#sendHeadRequest` passes
+     * a longer value on later retry attempts so a slow-but-reachable server
+     * gets another chance before being given up on.
+     */
+    readonly timeout?: number;
 }
 /**
  * Fetches the destination metadata for a URL using an HTTP HEAD request (or GET as fallback).
  *
  * Results are cached in memory so that repeated calls for the same URL
- * (without hash) return immediately. The request races against a 10-second
- * timeout; if the server does not respond in time, a {@link NetTimeoutError} is thrown.
+ * (without hash) return immediately. The request races against a configurable
+ * timeout (defaults to {@link DEFAULT_HEAD_TIMEOUT_MS}, 10 seconds); if the
+ * server does not respond in time, a {@link NetTimeoutError} is thrown.
  *
  * If the server returns 405 (Method Not Allowed), 501 (Not Implemented), or 503
  * (Service Unavailable) for a HEAD request, the function automatically retries with GET.
- * @param params - Parameters containing URL, external flag, method, options, and optional User-Agent.
+ * @param params - Parameters containing URL, external flag, method, options, optional User-Agent, and optional timeout.
  * @returns The page metadata obtained from the HTTP response.
- * @throws {NetTimeoutError} If the request exceeds the 10-second timeout.
+ * @throws {NetTimeoutError} If the request exceeds the configured timeout.
  * @throws {Error} If the HTTP request fails for any other reason.
  */
 export declare function fetchDestination(params: FetchDestinationParams): Promise<PageData>;

package/lib/crawler/fetch-destination.js

@@ -2,22 +2,26 @@ import { delay } from '@d-zero/shared/delay';
 import redirects from 'follow-redirects';
 import { destinationCache } from './destination-cache.js';
 import NetTimeoutError from './net-timeout-error.js';
+import { shouldGetFallbackOnHeadFailure } from './should-get-fallback-on-head-failure.js';
+/** Default race timeout for the HEAD pre-flight, in milliseconds. */
+const DEFAULT_HEAD_TIMEOUT_MS = 10 * 1000;
 /**
  * Fetches the destination metadata for a URL using an HTTP HEAD request (or GET as fallback).
  *
  * Results are cached in memory so that repeated calls for the same URL
- * (without hash) return immediately. The request races against a 10-second
- * timeout; if the server does not respond in time, a {@link NetTimeoutError} is thrown.
+ * (without hash) return immediately. The request races against a configurable
+ * timeout (defaults to {@link DEFAULT_HEAD_TIMEOUT_MS}, 10 seconds); if the
+ * server does not respond in time, a {@link NetTimeoutError} is thrown.
  *
  * If the server returns 405 (Method Not Allowed), 501 (Not Implemented), or 503
  * (Service Unavailable) for a HEAD request, the function automatically retries with GET.
- * @param params - Parameters containing URL, external flag, method, options, and optional User-Agent.
+ * @param params - Parameters containing URL, external flag, method, options, optional User-Agent, and optional timeout.
  * @returns The page metadata obtained from the HTTP response.
- * @throws {NetTimeoutError} If the request exceeds the 10-second timeout.
+ * @throws {NetTimeoutError} If the request exceeds the configured timeout.
  * @throws {Error} If the HTTP request fails for any other reason.
  */
 export async function fetchDestination(params) {
-    const { url, isExternal, method = 'HEAD', options, userAgent } = params;
+    const { url, isExternal, method = 'HEAD', options, userAgent, timeout } = params;
     const titleBytesLimit = options?.titleBytesLimit;
     const cacheKey = titleBytesLimit == null ? url.withoutHash : `${url.withoutHash}:title`;
     if (destinationCache.has(cacheKey)) {
@@ -28,20 +32,65 @@ export async function fetchDestination(params) {
         return cache;
     }
     const effectiveMethod = titleBytesLimit == null ? method : 'GET';
-    // Race the fetch against a 10-second timeout. The losing timer is cleared
+    const raceTimeoutMs = timeout ?? DEFAULT_HEAD_TIMEOUT_MS;
+    // Race the fetch against the requested timeout. The losing timer is cleared
     // explicitly so it never keeps the event loop alive after the race settles
     // (a plain `delay()` in `Promise.race` would leak the timer until it fires).
     let timeoutHandle;
     const result = await Promise.race([
-        _fetchHead(url, isExternal, effectiveMethod, titleBytesLimit, userAgent).catch((error) => (error instanceof Error ? error : new Error(String(error)))),
+        _fetchHead(url, isExternal, effectiveMethod, titleBytesLimit, userAgent, timeout).catch((error) => error instanceof Error ? error : new Error(String(error))),
         new Promise((resolve) => {
-            timeoutHandle = setTimeout(() => resolve(new NetTimeoutError(url.href)), 10 * 1000);
+            timeoutHandle = setTimeout(() => resolve(new NetTimeoutError(url.href)), raceTimeoutMs);
         }),
     ]).finally(() => {
         if (timeoutHandle)
             clearTimeout(timeoutHandle);
     });
-    destinationCache.set(cacheKey, result);
+    // HEAD failure fallback: a WAF / middlebox that silently drops HEAD will
+    // surface as NetTimeoutError / parse-error / connection-reset here even
+    // though the same URL serves a normal GET response. Try GET once (using
+    // the same timeout budget) before giving up on the URL. Only when
+    // `method === 'HEAD'` to avoid infinite recursion if the GET itself
+    // times out — at that point the server really is unreachable.
+    if (method === 'HEAD' &&
+        result instanceof Error &&
+        shouldGetFallbackOnHeadFailure(result)) {
+        try {
+            const getResult = await fetchDestination({
+                url,
+                isExternal,
+                method: 'GET',
+                userAgent,
+                timeout,
+            });
+            // GET succeeded — that is the canonical answer for this URL, so
+            // cache it under the HEAD cacheKey too (same key, since cacheKey
+            // only depends on URL + titleBytesLimit, not on method). The
+            // inner GET call already wrote to the cache under the same key,
+            // but a future caller hitting the HEAD path will find it there.
+            return getResult;
+        }
+        catch {
+            // GET fallback failed too; fall through to surface the original
+            // HEAD failure so retry / classification / DNS-burned cache see
+            // the actual underlying cause.
+        }
+    }
+    // Errors that are RECOVERABLE on retry — NetTimeoutError plus the kinds
+    // `shouldGetFallbackOnHeadFailure` already singles out as
+    // possibly-recoverable (parse-error, connection-reset) — are
+    // intentionally NOT cached. Caching a recoverable failure would freeze
+    // the first slow probe as the verdict for every later caller on the
+    // same host AND defeat `Crawler.#sendHeadRequest`'s
+    // HEAD_TIMEOUT_ESCALATION_MS (the 30s/60s retry would hit the cache and
+    // re-throw the stale 10s failure instead of getting the longer
+    // budget). DNS / TLS / refused / blocked are persistent within a crawl
+    // session so caching them is what keeps a doomed host from re-paying
+    // the network cost N times.
+    const isRecoverableError = result instanceof Error && shouldGetFallbackOnHeadFailure(result);
+    if (!isRecoverableError) {
+        destinationCache.set(cacheKey, result);
+    }
     if (result instanceof Error) {
         throw result;
     }
@@ -58,9 +107,11 @@ export async function fetchDestination(params) {
  * @param titleBytesLimit - When set, reads up to this many bytes from the response body
  *   to extract a `<title>` tag, then destroys the connection.
  * @param userAgent - Optional User-Agent string to send with the request.
+ * @param timeout - Optional race timeout in ms, forwarded to GET fallback so the
+ *   second pass keeps the same budget as the original HEAD attempt.
  * @returns A promise resolving to {@link PageData} with response metadata.
  */
-async function _fetchHead(url, isExternal, method, titleBytesLimit, userAgent) {
+async function _fetchHead(url, isExternal, method, titleBytesLimit, userAgent, timeout) {
     return new Promise((resolve, reject) => {
         const hostHeader = url.port ? `${url.hostname}:${url.port}` : url.hostname;
         // `trackRedirects` makes follow-redirects populate `res.redirects` with the
@@ -154,11 +205,21 @@ async function _fetchHead(url, isExternal, method, titleBytesLimit, userAgent) {
                     let rep = buildPageData('');
                     if (rep.status === 405) {
                         if (method === 'GET') {
-                            reject(new Error(`Method Not Allowed: ${url.href} ${rep.statusText}`));
+                            // GET fallback also returned 405 — the server really does
+                            // reject both methods. Resolve with the PageData so the
+                            // archive records `status: 405` instead of the `-1`
+                            // sentinel a reject would land on (which would erase the
+                            // only useful diagnostic the server gave us).
+                            resolve(rep);
                             return;
                         }
                         try {
-                            rep = await fetchDestination({ url, isExternal, method: 'GET' });
+                            rep = await fetchDestination({
+                                url,
+                                isExternal,
+                                method: 'GET',
+                                timeout,
+                            });
                         }
                         catch (error) {
                             reject(error);
@@ -167,12 +228,19 @@ async function _fetchHead(url, isExternal, method, titleBytesLimit, userAgent) {
                     }
                     if (rep.status === 501) {
                         if (method === 'GET') {
-                            reject(new Error(`Method Not Implemented: ${url.href} ${rep.statusText}`));
+                            // GET fallback also returned 501 — preserve the status
+                            // rather than dropping it into the `-1` bucket.
+                            resolve(rep);
                             return;
                         }
                         await delay(5 * 1000);
                         try {
-                            rep = await fetchDestination({ url, isExternal, method: 'GET' });
+                            rep = await fetchDestination({
+                                url,
+                                isExternal,
+                                method: 'GET',
+                                timeout,
+                            });
                         }
                         catch (error) {
                             reject(error);
@@ -181,12 +249,22 @@ async function _fetchHead(url, isExternal, method, titleBytesLimit, userAgent) {
                     }
                     if (rep.status === 503) {
                         if (method === 'GET') {
-                            reject(new Error(`Retrying failed: ${url.href} ${rep.statusText}`));
+                            // GET fallback also returned 503 — preserve the status.
+                            // A second-pass 5xx from a different method is the
+                            // server's real answer, not a transient HEAD-only quirk,
+                            // so the archive should remember it as 503 instead of
+                            // the generic `-1` sentinel.
+                            resolve(rep);
                             return;
                         }
                         await delay(5 * 1000);
                         try {
-                            rep = await fetchDestination({ url, isExternal, method: 'GET' });
+                            rep = await fetchDestination({
+                                url,
+                                isExternal,
+                                method: 'GET',
+                                timeout,
+                            });
                         }
                         catch (error) {
                             reject(error);

package/lib/crawler/is-js-redirect-error-shape.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Decide whether a browser-scrape error message is the specific
+ * `Page.goto() returned null` shape that the JS-redirect rescue is
+ * designed to recover from.
+ *
+ * **Why this gate exists:** before the gate, the rescue fired on *any*
+ * thrown error from `scraper.scrapeStart` as long as `page.url()` happened
+ * to report a different http(s) URL. That made every browser failure
+ * (TLS, target-crashed, OOM, navigation timeout, …) that incidentally
+ * left the page on a follow-up URL look like a JS redirect, hiding the
+ * real failure mode and stamping a phantom `status = 301` on the source.
+ *
+ * The narrow trigger only fires on the upstream's exact sentinel —
+ * `Page.goto returned null` — which beholder's scraper throws *only*
+ * when puppeteer's `page.goto()` resolved to `null`. Substring match (not
+ * equality) so wrapped variants like `[Retried 3 times] The method
+ * Page.goto returned null` (which surface in `crawl_errors` after retry
+ * exhaustion at outer layers) still classify, even though the rescue
+ * sees the bare form. Case-insensitive on the marker so a future
+ * beholder bump that lowercases the message keeps working.
+ *
+ * The trigger keys off the message *string*, not the message-classifier
+ * `kind`, because the rescue runs *before* the kind decision: the kind
+ * classifier would already wash this into `protocol`, and `protocol`
+ * covers more than just goto-null (Target closed / Session closed /
+ * detached Frame …) — none of which leave puppeteer with a meaningful
+ * post-navigation URL to recover.
+ * @param message - The raw error message from
+ *   `BrowserScrapeResult.error.message` (or any string that may carry
+ *   the sentinel inside a wrapper). `null` / `undefined` returns `false`.
+ * @returns `true` iff the message carries the `Page.goto returned null`
+ *   sentinel.
+ * @example
+ * ```ts
+ * isJsRedirectErrorShape('The method Page.goto returned null'); // → true
+ * isJsRedirectErrorShape('Navigation timeout of 60000 ms exceeded'); // → false
+ * isJsRedirectErrorShape(undefined); // → false
+ * ```
+ */
+export declare function isJsRedirectErrorShape(message: string | null | undefined): boolean;