@nitpicker/crawler 0.11.0 → 0.12.0

package/lib/crawler/crawler.js

@@ -3,16 +3,21 @@ import { existsSync } from 'node:fs';
 import path from 'node:path';
 import Scraper from '@d-zero/beholder';
 import { deal } from '@d-zero/dealer';
+import { delay } from '@d-zero/shared/delay';
 import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
 import { retryCall } from '@d-zero/shared/retry';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
 import c from 'ansi-colors';
 import pkg from '../../package.json' with { type: 'json' };
+import { classifyErrorKind } from '../classify-error-kind.js';
 import { crawlerLog } from '../debug.js';
+import { buildJsRedirectEdge } from './build-js-redirect-edge.js';
+import { buildRedirectEvent } from './build-redirect-event.js';
 import { createChangePhaseHandler } from './create-change-phase-handler.js';
 import { derivePageSource } from './derive-page-source.js';
-import { deriveResourceSource } from './derive-resource-source.js';
 import { detectPaginationPattern } from './detect-pagination-pattern.js';
+import { dnsBurnedHostCache } from './dns-burned-host-cache.js';
+import { dnsBurnedHostShortCircuitCounter } from './dns-burned-host-short-circuit-counter.js';
 import { drainPhaseErrors } from './drain-phase-errors.js';
 import { fetchDestination } from './fetch-destination.js';
 import { findScopeEntry } from './find-scope-entry.js';
@@ -20,21 +25,36 @@ import { formatCrawlProgress } from './format-crawl-progress.js';
 import { generatePredictedUrls } from './generate-predicted-urls.js';
 import { handleBrowserClose } from './handle-browser-close.js';
 import { handleIgnoreAndSkip } from './handle-ignore-and-skip.js';
-import { handleResourceResponse } from './handle-resource-response.js';
 import { handleScrapeEnd } from './handle-scrape-end.js';
 import { handleScrapeError } from './handle-scrape-error.js';
 import { injectScopeAuth } from './inject-scope-auth.js';
 import { isHtmlContentType } from './is-html-content-type.js';
+import { isLikelyHtmlUrl } from './is-likely-html-url.js';
+import { isPuppeteerFallbackCandidate } from './is-puppeteer-fallback-candidate.js';
 import LinkList from './link-list.js';
 import { linkToPageData } from './link-to-page-data.js';
 import { logUndrainedPhaseErrors } from './log-undrained-phase-errors.js';
 import { partitionUrlsByHtml } from './partition-urls-by-html.js';
+import { planSubResourceEmits } from './plan-sub-resource-emits.js';
+import { PreloadShortCircuitError } from './preload-short-circuit-error.js';
 import { protocolAgnosticKey } from './protocol-agnostic-key.js';
 import { redirectDestKey } from './redirect-dest-key.js';
 import { resourceToPageData } from './resource-to-page-data.js';
 import { RobotsChecker } from './robots-checker.js';
+import { shouldBurnHost } from './should-burn-host.js';
 import { shouldDiscardPredicted } from './should-discard-predicted.js';
 import { shouldSkipUrl } from './should-skip-url.js';
+/**
+ * Per-attempt HEAD pre-flight timeouts in milliseconds.
+ *
+ * `retryCall` re-invokes the work function up to `retry + 1` times; we keep
+ * the first attempt short so a fast healthy site never pays the slow-server
+ * tax, then escalate so that a slow-but-eventually-responsive host gets a
+ * larger budget on retry. The attempt index is clamped to the last element
+ * of the array, so configurations with `retry > escalation.length - 1` just
+ * stay on the final (longest) timeout for any additional attempts.
+ */
+const HEAD_TIMEOUT_ESCALATION_MS = [10_000, 30_000, 60_000];
 /**
  * The core crawler engine that discovers and scrapes web pages.
  *
@@ -54,7 +74,7 @@ class Crawler extends EventEmitter {
     /** Merged crawler configuration (user overrides + defaults). */
     #options;
     /**
-     * Phase errors observed during {@link Crawler.#launchBrowserAndScrape},
+     * Phase errors observed during {@link Crawler._launchBrowserAndScrape},
      * buffered per URL href so they can be emitted as `pageError` events
      * AFTER the corresponding `page` / `externalPage` event. This ordering
      * lets the orchestrator's WriteQueue serialise `setPage` before
@@ -80,6 +100,20 @@ class Crawler extends EventEmitter {
      * Keyed by {@link redirectDestKey}. Reset at the start of {@link #runDeal}.
      */
     #scrapedDestinations = new Set();
+    /**
+     * Lower-cased hostnames for which at least one URL has returned an
+     * HTTP response (any status) via `fetchDestination` in this session.
+     * Consulted by {@link shouldBurnHost} as the cascade guard against
+     * "transient local DNS hiccup wipes out a healthy host": a host that
+     * responded earlier is treated as still alive even when the next URL on
+     * it exhausts retries with a `getaddrinfo ENOTFOUND`, since the most
+     * likely cause is the operator's resolver flipping mid-crawl rather than
+     * the host suddenly disappearing. Populated by {@link #sendHeadRequest}
+     * on the success path; reset at the start of {@link #runDeal} alongside
+     * {@link #scrapedDestinations} so a fresh session does not inherit
+     * stale liveness assumptions.
+     */
+    #successfulHosts = new Set();
     /**
      * The AbortSignal associated with this crawler's AbortController.
      *
@@ -115,6 +149,7 @@ class Crawler extends EventEmitter {
             userAgent: options?.userAgent || `Nitpicker/${pkg.version}`,
             ignoreRobots: options?.ignoreRobots ?? false,
             lookupResource: options?.lookupResource ?? null,
+            lookupPageSource: options?.lookupPageSource ?? null,
             inventoryMode: options?.inventoryMode ?? null,
         };
         this.#robotsChecker = new RobotsChecker(this.#options.userAgent, !this.#options.ignoreRobots);
@@ -180,21 +215,27 @@ class Crawler extends EventEmitter {
      * before reaching the dealer so a URL that exists in both sources — which
      * is common in append-mode when a new root coincides with a repromoted
      * previously-external page — does not race on two parallel slots.
-     * @param urls - The list of root URLs to begin crawling from. Must be non-empty.
+     * @param urls - The list of root URLs to begin crawling from. May be empty
+     *   when resumed pending URLs already exist (for example `--retry-failed`).
      * @param opts - Optional overrides; currently only `recursive` is honoured.
      * @param opts.recursive - When `false`, disables recursive discovery and forces list-mode.
      *   Defaults to the constructor option's `recursive` value.
      * @throws {Error} If the URL list is empty.
      */
     start(urls, opts) {
-        const root = urls[0];
-        if (!root) {
-            throw new Error('urls is empty');
-        }
+        // Inventory mode pre-loads tens of thousands of seed URLs that all
+        // fall under archived `roots` (already populated into `#scope` by
+        // the constructor). Adding each seed as its own scope entry was
+        // O(N²) on build (per-host `existing.some` + array spread) AND
+        // turned every later `findScopeEntry` into a 70k linear scan. Skip
+        // the scope add — seeds remain entry points via `#linkList`.
+        const skipScopeAdd = this.#options.inventoryMode != null;
         for (const url of urls) {
-            const existing = this.#scope.get(url.hostname) || [];
-            if (!existing.some((u) => u.href === url.href)) {
-                this.#scope.set(url.hostname, [...existing, url]);
+            if (!skipScopeAdd) {
+                const existing = this.#scope.get(url.hostname) || [];
+                if (!existing.some((u) => u.href === url.href)) {
+                    this.#scope.set(url.hostname, [...existing, url]);
+                }
             }
             this.#linkList.add(url);
         }
@@ -223,13 +264,17 @@ class Crawler extends EventEmitter {
             seenInitial.add(key);
             initialUrls.push(url);
         }
+        const root = initialUrls[0];
+        if (!root) {
+            if (isResuming) {
+                crawlerLog('Crawl End (nothing to resume)');
+                void this.emit('crawlEnd', {});
+                return;
+            }
+            throw new Error('urls is empty');
+        }
         const resumeOffset = this.#resumedScraped.length;
         const pagesScrapedOffset = this.#resumedPagesScraped;
-        if (initialUrls.length === 0) {
-            crawlerLog('Crawl End (nothing to resume)');
-            void this.emit('crawlEnd', {});
-            return;
-        }
         void this.#runDeal(initialUrls, resumeOffset, pagesScrapedOffset).catch((error) => {
             crawlerLog('runDeal error: %O', error);
             this.#emitDealErrors(error, root.href);
@@ -281,25 +326,22 @@ class Crawler extends EventEmitter {
      * Processes captured sub-resources from a page scrape, deduplicates them,
      * and emits `response` / `responseReferrers` events for new resources.
      * @param resources - Sub-resource entries captured during the page load
+     * @param parentSource
      */
-    #handleResources(resources) {
-        // `deriveResourceSource` encodes the "sub-resources are never seeds"
-        // rule and stays in lockstep with `derivePageSource` if PageSource
-        // gains new variants. Computed once outside the loop because the
-        // inventoryMode reference does not change mid-batch.
-        const subResourceSource = deriveResourceSource(this.#options.inventoryMode);
-        for (const { resource, pageUrl } of resources) {
-            const { isNew } = handleResourceResponse(resource, this.#resources);
-            if (isNew) {
-                void this.emit('response', {
-                    resource: resource,
-                    source: subResourceSource,
-                });
-            }
-            void this.emit('responseReferrers', {
-                url: pageUrl,
-                src: resource.url.withoutHash,
-            });
+    #handleResources(resources, parentSource) {
+        // Decide the full emit plan first via the pure planner — that lets
+        // the lineage propagation contract (parent source → sub-resource
+        // `source`) be unit-tested in `plan-sub-resource-emits.spec.ts`
+        // without spinning up the puppeteer stack here. The previous
+        // inline shape made the `source` value invisible to tests because
+        // emit() side effects were only observable via a full scrape run
+        // that requires a mocked Chromium instance.
+        const { responseEmits, referrerEmits } = planSubResourceEmits(resources, parentSource, this.#resources);
+        for (const payload of responseEmits) {
+            void this.emit('response', payload);
+        }
+        for (const payload of referrerEmits) {
+            void this.emit('responseReferrers', payload);
         }
     }
     /**
@@ -440,72 +482,60 @@ class Crawler extends EventEmitter {
      * @param headCheckResult - Optional HEAD result to pass to the scraper, avoiding a redundant request
      * @returns The scrape result from beholder
      */
-    async #launchBrowserAndScrape(url, update, isExternal, metadataOnly, headCheckResult) {
-        update('Launching browser%dots%');
-        if (this.#options.executablePath) {
-            const execPath = path.resolve(this.#options.executablePath);
-            if (!existsSync(execPath)) {
-                throw new Error(`Executable path does not exist: ${execPath}`);
-            }
+    /**
+     * @param url
+     * @param update
+     * @param isExternal
+     * @param metadataOnly
+     * @param headCheckResult
+     * @internal
+     * cascade-guard contract for the puppeteer-fallback success / skipped
+     * branches can be exercised via `vi.spyOn(Crawler.prototype,
+     * '_launchBrowserAndScrape')` in unit tests. There is no production
+     * consumer outside this class.
+     */
+    /**
+     * Resolve the source label of the page being scraped so sub-resources
+     * captured during its render can inherit the correct lineage label
+     * (`'inventory-discovered'` when the parent is in the inventory chain,
+     * `undefined` otherwise so the DB DEFAULT `'crawled'` lands).
+     *
+     * Two-stage resolution:
+     *
+     * 1. If `inventoryMode` is active (live `--inventory` session), use
+     *    `derivePageSource` directly — the in-memory seed set is the
+     *    authoritative answer and no DB round-trip is needed.
+     *
+     * 2. Otherwise (`--resume`, `--retry-failed`, `--append`, or a normal
+     *    `crawl` of a previously-inventoried archive), ask the injected
+     *    `lookupPageSource` callback. The orchestrator wires that callback
+     *    to `Archive.getPageSourceByUrl` so the parent's lineage from
+     *    earlier sessions survives across sessions.
+     *
+     * One round-trip per page render at most — the result is not memoised
+     * because each worker scrapes a single page per `#scrapePage` call
+     * and the cost is amortised across every sub-resource of that page.
+     * @param url - The URL of the page being scraped.
+     * @returns The parent page's source, or `undefined` when none applies.
+     */
+    async #resolveParentSource(url) {
+        const fromInventoryMode = derivePageSource(this.#options.inventoryMode, url.withoutHashAndAuth);
+        if (fromInventoryMode !== undefined) {
+            return fromInventoryMode;
+        }
+        const lookupPageSource = this.#options.lookupPageSource;
+        if (!lookupPageSource) {
+            return undefined;
         }
-        const puppeteer = await import('puppeteer');
-        const browser = await puppeteer.launch({
-            headless: true,
-            ...(this.#options.executablePath
-                ? { executablePath: this.#options.executablePath }
-                : {}),
-        });
         try {
-            update('Creating page%dots%');
-            const page = await browser.newPage();
-            await page.setUserAgent(this.#options.userAgent);
-            // Defence-in-depth: beholder sets Authorization via setExtraHTTPHeaders,
-            // but page.authenticate() handles Chromium-level HTTP auth challenges
-            // (401 + WWW-Authenticate) that setExtraHTTPHeaders cannot cover.
-            if (url.username && url.password) {
-                await page.authenticate({
-                    username: url.username,
-                    password: url.password,
-                });
-            }
-            const scraper = new Scraper();
-            scraper.on('changePhase', createChangePhaseHandler({
-                emit: (event) => void this.emit('changePhase', event),
-                update,
-                formatLog: formatPhaseLog,
-                buffer: this.#pendingPhaseErrors,
-                urlHref: url.href,
-            }));
-            const result = await scraper.scrapeStart(page, url, {
-                isExternal,
-                captureImages: !isExternal && this.#options.captureImages,
-                excludeKeywords: this.#options.excludeKeywords,
-                disableQueries: this.#options.disableQueries,
-                metadataOnly,
-                retries: this.#options.retry,
-                headCheckResult,
-            });
-            update('Closing browser%dots%');
-            return result;
+            return await lookupPageSource(url.withoutHashAndAuth);
         }
         catch (error) {
-            return {
-                type: 'error',
-                resources: [],
-                error: {
-                    name: error instanceof Error ? error.name : 'Error',
-                    message: error instanceof Error ? error.message : String(error),
-                    stack: error instanceof Error ? error.stack : undefined,
-                    shutdown: true,
-                },
-            };
-        }
-        finally {
-            // handleBrowserClose force-kills the underlying Chromium when a
-            // graceful close() hangs (e.g. the session died mid-scrape) and
-            // guarantees the finally never throws, so the try-block's return
-            // value or caught error is never masked.
-            await handleBrowserClose(browser, url.href, crawlerLog);
+            // A lookup failure must never be worse than not having lineage
+            // — fall back to undefined so the sub-resources land at the DB
+            // DEFAULT `'crawled'` rather than crashing the whole worker.
+            crawlerLog('Parent source lookup failed for %s: %O', url.href, error);
+            return undefined;
         }
     }
     /**
@@ -530,6 +560,10 @@ class Crawler extends EventEmitter {
         }
         // Redirect-destination dedup is per-crawl; clear any state from a prior run.
         this.#scrapedDestinations.clear();
+        // Session-liveness signal is per-crawl too; clear so a fresh session
+        // does not inherit "host alive" claims from a prior run that may have
+        // happened on an entirely different network.
+        this.#successfulHosts.clear();
         // external URL の追跡（target は deal の total/done から導出）
         const externalUrls = new Set();
         const externalDoneUrls = new Set();
@@ -577,6 +611,20 @@ class Crawler extends EventEmitter {
                 return Promise.all(ops).then(() => { });
             };
             return async () => {
+                // Interval delay is handled here instead of by dealer because
+                // DNS-burned hosts must skip the wait entirely. Spending the
+                // per-URL interval on a host the cache already knows is dead
+                // just slows the crawl down for zero benefit — the HEAD won't
+                // be fired and `Crawler.#sendHeadRequest` will throw the
+                // preload short-circuit immediately. For all other URLs, run
+                // the same `delay()` + `%countdown(...)` log that dealer would
+                // have emitted, so the dealer display reads identically.
+                const burned = dnsBurnedHostCache.has(url.hostname.toLowerCase());
+                if (!burned && this.#options.interval && this.#options.interval > 0) {
+                    await delay(this.#options.interval, (determinedInterval) => {
+                        update(`Waiting interval: %countdown(${determinedInterval},${_index}_interval)%ms`);
+                    });
+                }
                 const log = createTimedUpdate(update, this.#options.verbose);
                 // `#scrapePage` 内のブラウザ HTML レンダーが成功したかをマークするフラグ。
                 // 成功時のみ #scrapePage 側で true に設定される。
@@ -640,8 +688,63 @@ class Crawler extends EventEmitter {
                         // path, where the first predicted source to a destination renders
                         // it and is recorded as a redirect source the same way; only 404 /
                         // error predicted URLs are dropped (by `shouldDiscardPredicted`).
-                        this.#linkList.done(url, this.#scope, { page: result.pageData }, this.#options);
-                        void this.emit('redirect', { result: result.pageData });
+                        //
+                        // The `source` discriminator divides this branch in two:
+                        //
+                        // - `'http-chain'` — the HEAD pre-flight resolved a real 3xx chain
+                        //   and the destination is already rendered (`#scrapedDestinations`
+                        //   claim). Every URL in `redirectPaths` is intermediate / known,
+                        //   so the existing behaviour applies: `linkList.done` folds the
+                        //   whole chain into the done-set so later references skip cleanly.
+                        //
+                        // - `'js-redirect'` — `scraper.scrapeStart` threw because
+                        //   `page.goto()` returned null (`window.location.replace()` /
+                        //   meta-refresh fired mid-navigation), and `redirectPaths`
+                        //   carries the single JS target Chromium ended up on. That target
+                        //   has NOT been rendered yet — it must enter the crawl queue, and
+                        //   `linkList.done` MUST NOT fold it into the done-set (otherwise
+                        //   the dealer's `seen` rejects the push and the destination is
+                        //   silently lost from the archive).
+                        if (result.source === 'js-redirect') {
+                            const destination = result.pageData.redirectPaths.at(-1);
+                            if (destination) {
+                                const destinationUrl = parseUrl(destination, this.#options);
+                                if (destinationUrl) {
+                                    this.#linkList.add(destinationUrl);
+                                    void enqueue(destinationUrl);
+                                }
+                                else {
+                                    // `deriveJsRedirectTarget` already canonicalises
+                                    // via WHATWG URL parsing, so reaching the
+                                    // `parseUrl === null` branch here would mean
+                                    // `@d-zero/shared/parse-url` rejected what
+                                    // WHATWG accepted — unexpected, and silently
+                                    // dropping the destination would be a silent
+                                    // archive loss. Log it so DEBUG=Nitpicker:Crawler
+                                    // catches the case.
+                                    crawlerLog('JS-redirect destination %s failed to parse — dropping enqueue', destination);
+                                }
+                            }
+                            else {
+                                crawlerLog('JS-redirect result for %s had no redirectPaths destination — dropping enqueue', url.href);
+                            }
+                            this.#linkList.done(url, this.#scope, { page: result.pageData }, this.#options, { includeRedirectPaths: false });
+                        }
+                        else {
+                            this.#linkList.done(url, this.#scope, { page: result.pageData }, this.#options);
+                        }
+                        // The redirect-edge call path may INSERT a brand-new
+                        // destination row (js-redirect rescue, #73
+                        // convergence on first sight). Forward the
+                        // originating page's inventory provenance so the
+                        // destination + intermediate hops inherit the
+                        // chain's lineage instead of laundering to DB
+                        // DEFAULT `'crawled'`. `inventoryMode === null`
+                        // (resume / retry-failed) yields `undefined`,
+                        // which is correct: the DB-side lookup in
+                        // `#linkRedirectSources` reads the destination's
+                        // stored source for those sessions.
+                        void this.emit('redirect', buildRedirectEvent(result.pageData, this.#options.inventoryMode, url.withoutHashAndAuth));
                         log(c.dim('Redirect (dest already scraped)'));
                         return;
                     }
@@ -660,7 +763,8 @@ class Crawler extends EventEmitter {
                     }
                     log('Saving results%dots%');
                     this.#handleResult(result, url, enqueue, paginationState, concurrency);
-                    this.#handleResources(result.resources);
+                    const parentSource = await this.#resolveParentSource(url);
+                    this.#handleResources(result.resources, parentSource);
                     log(formatResultSummary(result));
                     // Phase errors must be emitted AFTER 'page' / 'externalPage'
                     // so the orchestrator's WriteQueue sees `setPage` before
@@ -703,7 +807,10 @@ class Crawler extends EventEmitter {
             };
         }, {
             limit: concurrency,
-            interval: this.#options.interval,
+            // Interval is applied per-URL inside the worker callback above so
+            // DNS-burned hosts can skip it. Letting dealer handle interval
+            // would run the wait before our short-circuit check fires.
+            interval: 0,
             verbose: this.#options.verbose || !process.stdout.isTTY,
             signal: this.#abortController.signal,
             header: (_progress, done, total, limit) => {
@@ -745,7 +852,7 @@ class Crawler extends EventEmitter {
      * @param metadataOnly - When true, only extract title metadata without full browser scraping
      * @param laneIndex - The dealer lane index, used to create unique countdown IDs
      * @param markBrowserScrape - Called once **after** the browser successfully
-     *   renders an HTML page (i.e. `#launchBrowserAndScrape` resolved with
+     *   renders an HTML page (i.e. `_launchBrowserAndScrape` resolved with
      *   `type: 'success'`). Not called for HEAD-only, title-only, captured-resource
      *   reuse, non-HTML responses, non-HTTP protocols (mailto:, tel:), browser
      *   launch throws (e.g. invalid executablePath), or scraper-returned
@@ -757,7 +864,7 @@ class Crawler extends EventEmitter {
         const isExternal = findScopeEntry(url, this.#scope, this.#options) === null;
         // Non-HTTP protocols (mailto:, tel:, etc.) — let the scraper handle early return
         if (!url.isHTTP) {
-            return this.#launchBrowserAndScrape(url, update, isExternal, metadataOnly);
+            return this._launchBrowserAndScrape(url, update, isExternal, metadataOnly);
         }
         // Reuse captured resource data — when this URL was already observed as a
         // sub-resource during page rendering, its response data is recorded and
@@ -801,6 +908,131 @@ class Crawler extends EventEmitter {
             headCheckResult = await this.#sendHeadRequest(url, isExternal, update, laneIndex);
         }
         catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            // Puppeteer-only fallback: when the HEAD pre-flight (and its GET
+            // companion inside `fetchDestination`) exhaust retries on what
+            // looks like an HTML URL, give the browser exactly one chance
+            // before recording the page as `status = -1`. Some middleboxes /
+            // WAF configurations drop bare HEAD/GET probes (parse-error,
+            // reset, silent timeout) while still answering a real puppeteer
+            // navigation; those URLs would otherwise be permanently lost.
+            //
+            // Restricted to non-metadataOnly scrapes because metadata-only
+            // mode is a bandwidth-saving path for external pages — there is
+            // no payoff in spinning up puppeteer when the row was never
+            // going to be fully rendered. `isPuppeteerFallbackCandidate`
+            // filters PreloadShortCircuitError automatically via its
+            // classifier check (its synthesised message classifies as `dns`).
+            if (!metadataOnly &&
+                isLikelyHtmlUrl(url) &&
+                isPuppeteerFallbackCandidate(errorMessage)) {
+                update(c.yellow('HEAD/GET unreachable — trying puppeteer once'));
+                try {
+                    const fallback = await this._launchBrowserAndScrape(url, update, isExternal, metadataOnly);
+                    if (fallback.type === 'success') {
+                        if (fallback.pageData) {
+                            const renderedKey = redirectDestKey(url, fallback.pageData.redirectPaths);
+                            this.#scrapedDestinations.add(renderedKey);
+                        }
+                        // Puppeteer fallback proved the host is reachable
+                        // (HEAD/GET probes died at a middlebox / WAF but the
+                        // real browser navigation got a response). Mark the
+                        // host alive for the cascade guard — without this, a
+                        // host whose first URL only succeeded via the
+                        // browser-rescue path would still be vulnerable to
+                        // the next URL's HEAD failure burning it.
+                        this.#successfulHosts.add(url.hostname.toLowerCase());
+                        markBrowserScrape();
+                        return fallback;
+                    }
+                    if (fallback.type === 'skipped') {
+                        // Puppeteer rendered the page far enough for the scraper
+                        // to match an `excludeKeywords` rule. That is a definitive
+                        // "skip" verdict from the browser, NOT an unreachable
+                        // host — surface the skip so downstream handling (skip
+                        // counter, anchor-extraction suppression, `setSkippedPage`
+                        // in the archive) behaves identically to the case where
+                        // HEAD had succeeded. Without this branch, the page would
+                        // be recorded as `status = -1` with the HEAD timeout
+                        // message — a misleading entry that conflates
+                        // "operator-intended skip" with "network failure".
+                        //
+                        // Skipped also counts as proof-of-life: the browser
+                        // reached the page far enough to match exclude rules,
+                        // so the host was clearly responding.
+                        this.#successfulHosts.add(url.hostname.toLowerCase());
+                        return fallback;
+                    }
+                    // `fallback.type === 'error'`. `_launchBrowserAndScrape`
+                    // catches its own exceptions and returns
+                    // `{type:'error', shutdown:...}` rather than throwing, so
+                    // the `catch` arm below would NOT see this branch. Log
+                    // the puppeteer-side cause (and any `shutdown` flag the
+                    // scraper attached) so operators have a breadcrumb that
+                    // the safety net actually fired and lost — otherwise
+                    // only the HEAD error reaches `crawl_errors` and the
+                    // browser failure mode is invisible.
+                    crawlerLog('Puppeteer fallback returned error for %s: %s (shutdown=%s)', url.href, fallback.error?.message ?? '(no message)', fallback.error?.shutdown ?? false);
+                    // JS-redirect rescue on the puppeteer-fallback branch:
+                    // the HEAD/GET probes died (the kind set in
+                    // `isPuppeteerFallbackCandidate` — middlebox / WAF
+                    // shapes), the one-shot puppeteer attempt also threw,
+                    // but `page.url()` reported a different post-navigation
+                    // URL. This is the same WAF-+-JS-redirect shape the
+                    // HEAD-success rescue handles one branch below, applied
+                    // to the prior failure layer. Without this, a URL whose
+                    // only sin is "HEAD blocked + JS-redirected body" falls
+                    // to `status = -1` and joins the retry-forever loop the
+                    // rescue is supposed to break. The trigger is the same
+                    // narrow `Page.goto returned null` shape — anything
+                    // else (TLS failure inside puppeteer, target crash, …)
+                    // must fall through to the unreachable path so the real
+                    // failure surfaces. We synthesise the redirect-edge
+                    // PageData from the HEAD error (status = -1) instead of
+                    // from a HEAD success, so `#linkRedirectSources` still
+                    // stamps the source as 301 and the edge wires the dest
+                    // in.
+                    const fallbackRescue = buildJsRedirectEdge({
+                        url,
+                        isExternal,
+                        errorMessage: fallback.error?.message,
+                        postNavigationUrl: fallback.postNavigationUrl,
+                        // No `headCheckResult`: HEAD itself died on this
+                        // path, so the synthesised PageData starts from a
+                        // `linkToPageData` placeholder with `status = -1`
+                        // carrying the original HEAD error message.
+                        // `#linkRedirectSources` still flips the source row
+                        // to 301 because NULL/-1 satisfies its conditional
+                        // stamp predicate.
+                    });
+                    if (fallbackRescue !== null) {
+                        return fallbackRescue;
+                    }
+                }
+                catch (browserError) {
+                    // Browser launch / runtime crash — fall through to the
+                    // unreachable path below. The original HEAD error is more
+                    // informative about WHY the URL wasn't reachable, so it
+                    // (not the puppeteer noise) is what we surface in
+                    // `crawl_errors`. The lane display flag below (
+                    // "Unreachable (fallback failed)") preserves the fact
+                    // that puppeteer also tried, so operators reading the
+                    // progress log can tell this URL got the safety-net
+                    // attempt versus the cheap-probe-only path.
+                    crawlerLog('Puppeteer fallback also failed for %s: %O', url.href, browserError);
+                }
+                update(c.red('Unreachable (fallback failed)'));
+                return {
+                    type: 'error',
+                    resources: [],
+                    error: {
+                        name: error instanceof Error ? error.name : 'Error',
+                        message: errorMessage,
+                        stack: error instanceof Error ? error.stack : undefined,
+                        shutdown: false,
+                    },
+                };
+            }
             // Server unreachable — skip browser launch entirely
             update(c.red('Unreachable'));
             return {
@@ -808,7 +1040,7 @@ class Crawler extends EventEmitter {
                 resources: [],
                 error: {
                     name: error instanceof Error ? error.name : 'Error',
-                    message: error instanceof Error ? error.message : String(error),
+                    message: errorMessage,
                     stack: error instanceof Error ? error.stack : undefined,
                     shutdown: false,
                 },
@@ -834,7 +1066,7 @@ class Crawler extends EventEmitter {
         const finalKey = redirectDestKey(url, headCheckResult.redirectPaths);
         if (this.#scrapedDestinations.has(finalKey)) {
             crawlerLog('Redirect dest already rendered, edge only: %s', url.href);
-            return { type: 'redirect-edge', pageData: headCheckResult };
+            return { type: 'redirect-edge', source: 'http-chain', pageData: headCheckResult };
         }
         // Title-only mode — extract <title> via partial GET for HTML, skip browser
         if (metadataOnly) {
@@ -876,12 +1108,12 @@ class Crawler extends EventEmitter {
         }
         // HTML or unknown content type — launch browser with preflight result.
         // markBrowserScrape() fires only when the result is `success`.
-        // `#launchBrowserAndScrape` catches internal errors and returns
+        // `_launchBrowserAndScrape` catches internal errors and returns
         // `{ type: 'error', ... }` instead of throwing (see its catch block),
         // so awaiting alone does NOT prove the page was rendered. The explicit
         // success check excludes navigation failures, scraper exceptions, and
         // shutdown-class errors from the pages-rendered count.
-        const browserResult = await this.#launchBrowserAndScrape(url, update, isExternal, metadataOnly, headCheckResult);
+        const browserResult = await this._launchBrowserAndScrape(url, update, isExternal, metadataOnly, headCheckResult);
         if (browserResult.type === 'success') {
             markBrowserScrape();
             // Claim the destination that was ACTUALLY rendered, keyed off the
@@ -905,6 +1137,95 @@ class Crawler extends EventEmitter {
                 ? redirectDestKey(url, browserResult.pageData.redirectPaths)
                 : finalKey;
             this.#scrapedDestinations.add(renderedKey);
+            return browserResult;
+        }
+        // Browser scrape failed but the HEAD pre-flight already resolved a
+        // redirect chain — fall back to the redirect-edge path so the chain
+        // is not lost. Without this, a URL whose final destination is on
+        // HTTPS→HTTP downgrade (or any other navigation Chromium refuses
+        // to complete while the underlying redirect was a normal 301/302)
+        // would be persisted as `status = -1` with NULL `redirectDestId`,
+        // then re-picked up by every `--retry-failed` pass forever — the
+        // HEAD answer is the authoritative truth and the browser cannot
+        // invalidate it.
+        //
+        // Restricted to `type === 'error'` because:
+        // - `'skipped'` is an `excludeKeywords` verdict from the browser
+        //   on the rendered URL and is its own definitive outcome —
+        //   surfacing it as a redirect-edge would lose the skip signal.
+        // - `'success'` is handled above.
+        //
+        // The destination is claimed even though no row was rendered for
+        // it: subsequent siblings on the same chain should also fold into
+        // the same edge instead of re-firing the same failing browser
+        // attempt. If the destination URL itself reaches the queue later,
+        // it goes through the normal `#scrapePage` path (the claim only
+        // short-circuits sibling redirect SOURCES, not the destination
+        // itself).
+        if (browserResult.type === 'error' && headCheckResult.redirectPaths.length > 0) {
+            this.#scrapedDestinations.add(finalKey);
+            crawlerLog('Browser scrape failed for %s but HEAD resolved a redirect chain — recording as edge', url.href);
+            return { type: 'redirect-edge', source: 'http-chain', pageData: headCheckResult };
+        }
+        // JS-redirect rescue: HEAD returned a definitive response (no chain),
+        // the browser scrape threw with the specific `Page.goto returned null`
+        // shape (gated by `isJsRedirectErrorShape` below), and puppeteer
+        // reports a different post-navigation URL via `page.url()`. The
+        // motivating case is a server returning `200 OK` whose body contains
+        // `window.location.replace(...)` or `<meta http-equiv="refresh">` —
+        // `page.goto()` resolves to `null` once the JS-driven navigation
+        // supersedes the original, and the scraper throws
+        // `The method Page.goto returned null`. Recording the edge preserves
+        // the link from the source to the JS-redirect target, removes the
+        // page from `--retry-failed`'s candidate pool (the SQL filter
+        // excludes rows with a non-null `redirectDestId`), and matches what
+        // a real browser shows the user.
+        //
+        // What the source row reads as:
+        // - the source is not committed via `setPage`/`updatePage` on this
+        //   path (the redirect-edge handler in `#runDeal` only calls
+        //   `linkList.done` + `emit('redirect', ...)` → `Archive.setRedirect`),
+        //   so `recordRedirect` → `#getIdByUrl` creates a NULL-status
+        //   placeholder row for the source if it did not already exist;
+        // - `#linkRedirectSources` then stamps `status = 301
+        //   statusText='Moved Permanently'` because NULL satisfies its
+        //   conditional-update predicate.
+        // That is the same shape an HTTP 301 source ends up with — the
+        // truthful HTTP layer (the upstream's 200) is lost on this path, but
+        // the alternative (status=-1 retry-forever) is strictly worse. A
+        // future refinement could keep the HEAD-derived status by routing
+        // the source through `setPage` before `setRedirect`; intentionally
+        // deferred to keep this rescue minimal.
+        //
+        // Pre-claiming the destination in `#scrapedDestinations` would
+        // short-circuit the freshly-enqueued destination at the top of
+        // `#scrapePage` (the `if (#scrapedDestinations.has(finalKey))` guard
+        // at line 1213), leaving the dest row as a content-less HEAD edge
+        // instead of a fully rendered page. So we *do not* claim here — the
+        // destination renders normally via the queue, and `#scrapedDestinations`
+        // is populated at line ~1322 of the render-success path the way every
+        // other URL is. Sibling JS-redirect sources to the same destination
+        // still converge: the second sibling enters this branch, observes its
+        // own `page.url()` landing on the same target, records its own
+        // redirect-edge, and re-enqueues — the dealer's `seen` dedup absorbs
+        // the duplicate push, so the destination renders exactly once.
+        if (browserResult.type === 'error') {
+            const headSuccessRescue = buildJsRedirectEdge({
+                url,
+                isExternal,
+                errorMessage: browserResult.error?.message,
+                postNavigationUrl: browserResult.postNavigationUrl,
+                // `headCheckResult` is supplied here so the synthesised
+                // PageData carries the real HTTP-level status / content
+                // type from the HEAD pre-flight. `#linkRedirectSources`
+                // only stamps 301 onto NULL/-1 status rows, so the
+                // HEAD-derived status DOES survive on this path — the
+                // truthful HTTP 200 is preserved.
+                headCheckResult,
+            });
+            if (headSuccessRescue !== null) {
+                return headSuccessRescue;
+            }
         }
         return browserResult;
     }
@@ -921,17 +1242,244 @@ class Crawler extends EventEmitter {
      * @returns Lightweight page data from the HEAD response
      */
     async #sendHeadRequest(url, isExternal, update, laneIndex) {
-        return retryCall(() => fetchDestination({ url, isExternal, userAgent: this.#options.userAgent }), {
+        const host = url.hostname.toLowerCase();
+        if (dnsBurnedHostCache.has(host)) {
+            // Either session-learned earlier in this crawl (one URL on this host
+            // already exhausted retries with a DNS error) or preload-seeded from
+            // `crawl_errors` on archive open. Either way: skip the HEAD entirely.
+            // The orchestrator's error-channel listener detects
+            // PreloadShortCircuitError via instanceof and refuses to write it to
+            // `crawl_errors`, preventing self-amplification across crawls.
+            dnsBurnedHostShortCircuitCounter.count++;
+            update(c.red(`HEAD request: host ${host} DNS-burned — skipping`));
+            throw new PreloadShortCircuitError(host);
+        }
+        // Escalating per-attempt timeout: a slow-but-reachable server (e.g. some
+        // government sites under load) often answers in 20-40 s but is missed by
+        // a flat 10 s race on every retry. Start short to keep crawl throughput
+        // up on healthy URLs, then back off so the last attempt is generous
+        // enough that "really slow" gets a fair shot before we give up.
+        let attempt = 0;
+        return retryCall(async () => {
+            // Clamp the attempt index to the last entry of the escalation array
+            // so retry counts past the array length keep using the longest
+            // budget instead of falling off into `undefined`. `as number`
+            // only because TS can't see that a positive-length readonly array
+            // always has a defined last element.
+            const escalationIndex = Math.min(attempt, HEAD_TIMEOUT_ESCALATION_MS.length - 1);
+            const timeoutMs = HEAD_TIMEOUT_ESCALATION_MS[escalationIndex];
+            attempt += 1;
+            const headResult = await fetchDestination({
+                url,
+                isExternal,
+                userAgent: this.#options.userAgent,
+                timeout: timeoutMs,
+            });
+            // Mark host alive the MOMENT an HTTP response is observed,
+            // before retryCall's outer resolution settles. A later attempt
+            // (or a sibling worker's onGiveUp) racing this success would
+            // otherwise see an empty `#successfulHosts` and burn the host
+            // — exactly the cascade the guard is here to prevent. Any HTTP
+            // status counts: the guard cares about DNS-and-TCP reachability,
+            // not application-level success, and `fetchDestination` only
+            // resolves when an HTTP response was actually received.
+            this.#successfulHosts.add(host);
+            return headResult;
+        }, {
             retries: this.#options.retry,
             label: 'HEAD request',
             onWait: (determinedInterval, retryCount, label, error) => {
                 update(`${label}: ${error.message} — %countdown(${determinedInterval},fetchHead_${laneIndex}_${retryCount},s)%s (retry #${retryCount + 1})`);
             },
             onGiveUp: (retryCount, error, label) => {
+                // Burn the host so subsequent URLs short-circuit — but ONLY
+                // when this is the first time we've ever seen the host fail
+                // in this session. A host that responded earlier is treated
+                // as transiently unreachable (operator's resolver flipped
+                // mid-crawl etc.), not a dead domain. `shouldBurnHost`
+                // encapsulates this decision so the cascade guard is
+                // independently testable. Also gated to `onGiveUp` rather
+                // than `onWait` so an `EAI_AGAIN` that recovers on retry
+                // doesn't trip the guard prematurely.
+                if (shouldBurnHost({
+                    errorKind: classifyErrorKind(error.message),
+                    host,
+                    successfulHosts: this.#successfulHosts,
+                })) {
+                    dnsBurnedHostCache.set(host, 'dns');
+                }
                 update(c.red(`${label}: gave up after ${retryCount} retries — ${error.message}`));
             },
         });
     }
+    // eslint-disable-next-line no-restricted-syntax -- intentional `private` (vs `#`) so tests can spyOn the prototype to drive the puppeteer-fallback cascade-guard branches without a full browser mock; see JSDoc above.
+    async _launchBrowserAndScrape(url, update, isExternal, metadataOnly, headCheckResult) {
+        update('Launching browser%dots%');
+        if (this.#options.executablePath) {
+            const execPath = path.resolve(this.#options.executablePath);
+            if (!existsSync(execPath)) {
+                throw new Error(`Executable path does not exist: ${execPath}`);
+            }
+        }
+        const puppeteer = await import('puppeteer');
+        const browser = await puppeteer.launch({
+            headless: true,
+            ...(this.#options.executablePath
+                ? { executablePath: this.#options.executablePath }
+                : {}),
+        });
+        // `page` is hoisted out of the try-block so the catch arm can read
+        // `page.url()` for JS-redirect detection. See `BrowserScrapeResult`
+        // JSDoc for the full why; in short, when `scrapeStart` throws because
+        // `page.goto()` returned `null`, the puppeteer page object still
+        // holds the URL Chromium actually navigated to via the offending
+        // `window.location.replace()` / meta-refresh, and that is the only
+        // authoritative source for the JS-redirect destination.
+        let page = null;
+        try {
+            update('Creating page%dots%');
+            page = await browser.newPage();
+            await page.setUserAgent(this.#options.userAgent);
+            // HTTP-auth handling — two cooperating pieces, BOTH required:
+            //
+            // 1. `page.authenticate({user, pass})` (always, even with empty
+            //    strings) registers a Fetch-domain auth handler with
+            //    Chromium. With empty credentials it ALSO drains Chromium's
+            //    native HTTP-auth dialog without sending anything
+            //    privileged — the dialog cannot be captured by
+            //    `page.on('dialog')` (HTTP-auth is not a JS dialog) and
+            //    would otherwise hang the navigation until puppeteer's
+            //    timeout fires. With non-empty credentials it provides the
+            //    scope's auth so the in-scope navigation succeeds.
+            //
+            // 2. Stripping URL-embedded credentials from the navigation
+            //    target. **This is the credential-leak guard.** When the
+            //    URL we hand puppeteer carries `user:pass@host`, Chromium
+            //    promotes those credentials into its HTTP-auth cache
+            //    keyed by (scheme, host, port, realm). Subsequent
+            //    sub-resource requests issued from the same page —
+            //    including cross-origin requests to a different hostname
+            //    sharing the same IP / port (e.g. an embedded
+            //    `<img src="http://127.0.0.1:8010/…">` loaded from a
+            //    `localhost:8010` page) — get the cached `Authorization`
+            //    header re-attached by the network stack. The
+            //    `Fetch.authRequired` event never fires for these
+            //    pre-emptive attachments, so neither `page.authenticate`
+            //    nor any custom Fetch listener can filter them. The only
+            //    way to keep the cred out of the cross-origin request is
+            //    to make sure it never enters the cache in the first
+            //    place — hence stripping the URL before navigation.
+            //
+            // Verified by `scope-auth-leak.e2e.ts`: removing either piece
+            // causes that test to fail (without auth → main 401 hangs;
+            // without strip → scope cred leaks to off-scope sub-resource).
+            await page.authenticate({
+                username: url.username ?? '',
+                password: url.password ?? '',
+            });
+            // Re-parse from `withoutHashAndAuth` rather than mutating the
+            // re-parsed `url.href` object: ExURL pre-computes `href`,
+            // `withoutHash` and other derived strings at parse time, and
+            // post-hoc field assignment (`navigateUrl.username = ''`)
+            // leaves those derived strings stale. Anything downstream that
+            // reads `navigateUrl.href` (e.g. a future beholder bump that
+            // switches `page.goto` from `withoutHashAndAuth` to `href`)
+            // would silently get back the credentialed string — defeating
+            // the leak guard. Building the navigation URL from a known
+            // credential-free string guarantees every field is consistent.
+            const navigateUrl = parseUrl(url.withoutHashAndAuth) ?? url;
+            const scraper = new Scraper();
+            scraper.on('changePhase', createChangePhaseHandler({
+                emit: (event) => void this.emit('changePhase', event),
+                update,
+                formatLog: formatPhaseLog,
+                buffer: this.#pendingPhaseErrors,
+                urlHref: url.href,
+            }));
+            const result = await scraper.scrapeStart(page, navigateUrl, {
+                isExternal,
+                captureImages: !isExternal && this.#options.captureImages,
+                excludeKeywords: this.#options.excludeKeywords,
+                disableQueries: this.#options.disableQueries,
+                metadataOnly,
+                retries: this.#options.retry,
+                headCheckResult,
+            });
+            update('Closing browser%dots%');
+            // JS-redirect rescue capture: when `scrapeStart` catches a
+            // `#fetchData` throw internally (e.g. `Page.goto returned null`
+            // because a client-side `window.location.replace()` /
+            // meta-refresh fired), it returns `{ type: 'error', ... }`
+            // instead of re-throwing — so the `catch` arm below never
+            // sees those cases. Read `page.url()` here while `page` is
+            // still alive (finally still hasn't called `handleBrowserClose`)
+            // and attach it to the result so `#scrapePage` can fold the
+            // source into a redirect edge. Without this capture, the
+            // rescue path is dead for the most common failure shape it
+            // was designed to handle.
+            //
+            // `page.url()` itself can throw when the browser context died
+            // mid-scrape (target crashed, session killed). On failure we
+            // fall through with `postNavigationUrl` unset so the existing
+            // HEAD-chain rescue / normal error path takes over.
+            if (result.type === 'error') {
+                try {
+                    const postNavigationUrl = page.url();
+                    return { ...result, postNavigationUrl };
+                }
+                catch (urlReadError) {
+                    crawlerLog('Reading page.url() for JS-redirect detection failed on %s: %O', url.href, urlReadError);
+                }
+            }
+            return result;
+        }
+        catch (error) {
+            // JS-redirect rescue: when `scrapeStart` throws because
+            // `page.goto()` returned `null` (the symptom of a client-side
+            // `window.location.replace()` / meta-refresh navigating away
+            // before the original response materialised), `page.url()` still
+            // reports the destination Chromium ended up on. Capturing it
+            // here lets `#scrapePage` fold the source into a redirect edge
+            // instead of recording a hard `status = -1` — `Page.goto returned
+            // null` classifies as `protocol`, which is neither permanent nor
+            // a puppeteer-fallback kind, so without this rescue the page
+            // loops through `--retry-failed` forever with the same failure.
+            //
+            // `page.url()` itself can throw when the browser context is
+            // already torn down (target closed, session killed). Treat any
+            // such failure as "no extra information" and fall back to the
+            // normal error path — the existing redirect-edge fallback that
+            // keys off `headCheckResult.redirectPaths` may still rescue the
+            // page when the HEAD pre-flight resolved a chain.
+            let postNavigationUrl;
+            if (page) {
+                try {
+                    postNavigationUrl = page.url();
+                }
+                catch (urlReadError) {
+                    crawlerLog('Reading page.url() for JS-redirect detection failed on %s: %O', url.href, urlReadError);
+                }
+            }
+            return {
+                type: 'error',
+                resources: [],
+                error: {
+                    name: error instanceof Error ? error.name : 'Error',
+                    message: error instanceof Error ? error.message : String(error),
+                    stack: error instanceof Error ? error.stack : undefined,
+                    shutdown: true,
+                },
+                ...(postNavigationUrl === undefined ? {} : { postNavigationUrl }),
+            };
+        }
+        finally {
+            // handleBrowserClose force-kills the underlying Chromium when a
+            // graceful close() hangs (e.g. the session died mid-scrape) and
+            // guarantees the finally never throws, so the try-block's return
+            // value or caught error is never masked.
+            await handleBrowserClose(browser, url.href, crawlerLog);
+        }
+    }
     /**
      * The default maximum number of concurrent scraping processes.
      *