@nitpicker/crawler 0.9.0 → 0.12.0

package/lib/crawler/crawler.js

@@ -3,28 +3,58 @@ import { existsSync } from 'node:fs';
 import path from 'node:path';
 import Scraper from '@d-zero/beholder';
 import { deal } from '@d-zero/dealer';
+import { delay } from '@d-zero/shared/delay';
 import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
 import { retryCall } from '@d-zero/shared/retry';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
 import c from 'ansi-colors';
 import pkg from '../../package.json' with { type: 'json' };
+import { classifyErrorKind } from '../classify-error-kind.js';
 import { crawlerLog } from '../debug.js';
+import { buildJsRedirectEdge } from './build-js-redirect-edge.js';
+import { buildRedirectEvent } from './build-redirect-event.js';
+import { createChangePhaseHandler } from './create-change-phase-handler.js';
+import { derivePageSource } from './derive-page-source.js';
 import { detectPaginationPattern } from './detect-pagination-pattern.js';
+import { dnsBurnedHostCache } from './dns-burned-host-cache.js';
+import { dnsBurnedHostShortCircuitCounter } from './dns-burned-host-short-circuit-counter.js';
+import { drainPhaseErrors } from './drain-phase-errors.js';
 import { fetchDestination } from './fetch-destination.js';
 import { findScopeEntry } from './find-scope-entry.js';
 import { formatCrawlProgress } from './format-crawl-progress.js';
 import { generatePredictedUrls } from './generate-predicted-urls.js';
+import { handleBrowserClose } from './handle-browser-close.js';
 import { handleIgnoreAndSkip } from './handle-ignore-and-skip.js';
-import { handleResourceResponse } from './handle-resource-response.js';
 import { handleScrapeEnd } from './handle-scrape-end.js';
 import { handleScrapeError } from './handle-scrape-error.js';
 import { injectScopeAuth } from './inject-scope-auth.js';
+import { isHtmlContentType } from './is-html-content-type.js';
+import { isLikelyHtmlUrl } from './is-likely-html-url.js';
+import { isPuppeteerFallbackCandidate } from './is-puppeteer-fallback-candidate.js';
 import LinkList from './link-list.js';
 import { linkToPageData } from './link-to-page-data.js';
+import { logUndrainedPhaseErrors } from './log-undrained-phase-errors.js';
+import { partitionUrlsByHtml } from './partition-urls-by-html.js';
+import { planSubResourceEmits } from './plan-sub-resource-emits.js';
+import { PreloadShortCircuitError } from './preload-short-circuit-error.js';
 import { protocolAgnosticKey } from './protocol-agnostic-key.js';
+import { redirectDestKey } from './redirect-dest-key.js';
+import { resourceToPageData } from './resource-to-page-data.js';
 import { RobotsChecker } from './robots-checker.js';
+import { shouldBurnHost } from './should-burn-host.js';
 import { shouldDiscardPredicted } from './should-discard-predicted.js';
 import { shouldSkipUrl } from './should-skip-url.js';
+/**
+ * Per-attempt HEAD pre-flight timeouts in milliseconds.
+ *
+ * `retryCall` re-invokes the work function up to `retry + 1` times; we keep
+ * the first attempt short so a fast healthy site never pays the slow-server
+ * tax, then escalate so that a slow-but-eventually-responsive host gets a
+ * larger budget on retry. The attempt index is clamped to the last element
+ * of the array, so configurations with `retry > escalation.length - 1` just
+ * stay on the final (longest) timeout for any additional attempts.
+ */
+const HEAD_TIMEOUT_ESCALATION_MS = [10_000, 30_000, 60_000];
 /**
  * The core crawler engine that discovers and scrapes web pages.
  *
@@ -43,8 +73,18 @@ class Crawler extends EventEmitter {
     #linkList = new LinkList();
     /** Merged crawler configuration (user overrides + defaults). */
     #options;
+    /**
+     * Phase errors observed during {@link Crawler._launchBrowserAndScrape},
+     * buffered per URL href so they can be emitted as `pageError` events
+     * AFTER the corresponding `page` / `externalPage` event. This ordering
+     * lets the orchestrator's WriteQueue serialise `setPage` before
+     * `insertPageError`, so the FK resolution via URL always finds the row.
+     */
+    #pendingPhaseErrors = new Map();
     /** Set of resource URLs (without hash) already captured, for deduplication. */
     #resources = new Set();
+    /** Number of HTML pages (isTarget=1) scraped in previous sessions, used to seed the progress counter on resume. */
+    #resumedPagesScraped = 0;
     /** URLs restored from a previous session that still need to be scraped. */
     #resumedPending = [];
     /** URLs already scraped in a previous session, used to populate the `seen` set in {@link #runDeal}. */
@@ -53,6 +93,27 @@ class Crawler extends EventEmitter {
     #robotsChecker;
     /** Maps hostnames to their scope URLs. Defines the crawl boundary for internal/external classification. */
     #scope = new Map();
+    /**
+     * Protocol-agnostic keys of redirect destinations already rendered (and stored)
+     * during this crawl. When many URLs redirect to one destination, only the first
+     * renders it; the rest record the redirect edge and skip the browser (#73).
+     * Keyed by {@link redirectDestKey}. Reset at the start of {@link #runDeal}.
+     */
+    #scrapedDestinations = new Set();
+    /**
+     * Lower-cased hostnames for which at least one URL has returned an
+     * HTTP response (any status) via `fetchDestination` in this session.
+     * Consulted by {@link shouldBurnHost} as the cascade guard against
+     * "transient local DNS hiccup wipes out a healthy host": a host that
+     * responded earlier is treated as still alive even when the next URL on
+     * it exhausts retries with a `getaddrinfo ENOTFOUND`, since the most
+     * likely cause is the operator's resolver flipping mid-crawl rather than
+     * the host suddenly disappearing. Populated by {@link #sendHeadRequest}
+     * on the success path; reset at the start of {@link #runDeal} alongside
+     * {@link #scrapedDestinations} so a fresh session does not inherit
+     * stale liveness assumptions.
+     */
+    #successfulHosts = new Set();
     /**
      * The AbortSignal associated with this crawler's AbortController.
      *
@@ -87,6 +148,9 @@ class Crawler extends EventEmitter {
             verbose: options?.verbose ?? false,
             userAgent: options?.userAgent || `Nitpicker/${pkg.version}`,
             ignoreRobots: options?.ignoreRobots ?? false,
+            lookupResource: options?.lookupResource ?? null,
+            lookupPageSource: options?.lookupPageSource ?? null,
+            inventoryMode: options?.inventoryMode ?? null,
         };
         this.#robotsChecker = new RobotsChecker(this.#options.userAgent, !this.#options.ignoreRobots);
         for (const urlStr of this.#options.roots) {
@@ -126,10 +190,14 @@ class Crawler extends EventEmitter {
      * @param pending - URLs that were pending (not yet scraped) in the previous session.
      * @param scraped - URLs that were already scraped in the previous session.
      * @param resources - Resource URLs that were already captured in the previous session.
+     * @param pagesScrapedOffset - Number of HTML pages already rendered in previous
+     *   sessions, used to seed the session-spanning progress counter. Defaults to 0
+     *   for callers that don't need cross-session accuracy in the progress display.
      */
-    resume(pending, scraped, resources) {
+    resume(pending, scraped, resources, pagesScrapedOffset = 0) {
         this.#resumedPending = this.#linkList.resume(pending, scraped, this.#options);
         this.#resumedScraped = scraped;
+        this.#resumedPagesScraped = pagesScrapedOffset;
         for (const resource of resources) {
             this.#resources.add(resource);
         }
@@ -147,21 +215,27 @@ class Crawler extends EventEmitter {
      * before reaching the dealer so a URL that exists in both sources — which
      * is common in append-mode when a new root coincides with a repromoted
      * previously-external page — does not race on two parallel slots.
-     * @param urls - The list of root URLs to begin crawling from. Must be non-empty.
+     * @param urls - The list of root URLs to begin crawling from. May be empty
+     *   when resumed pending URLs already exist (for example `--retry-failed`).
      * @param opts - Optional overrides; currently only `recursive` is honoured.
      * @param opts.recursive - When `false`, disables recursive discovery and forces list-mode.
      *   Defaults to the constructor option's `recursive` value.
      * @throws {Error} If the URL list is empty.
      */
     start(urls, opts) {
-        const root = urls[0];
-        if (!root) {
-            throw new Error('urls is empty');
-        }
+        // Inventory mode pre-loads tens of thousands of seed URLs that all
+        // fall under archived `roots` (already populated into `#scope` by
+        // the constructor). Adding each seed as its own scope entry was
+        // O(N²) on build (per-host `existing.some` + array spread) AND
+        // turned every later `findScopeEntry` into a 70k linear scan. Skip
+        // the scope add — seeds remain entry points via `#linkList`.
+        const skipScopeAdd = this.#options.inventoryMode != null;
         for (const url of urls) {
-            const existing = this.#scope.get(url.hostname) || [];
-            if (!existing.some((u) => u.href === url.href)) {
-                this.#scope.set(url.hostname, [...existing, url]);
+            if (!skipScopeAdd) {
+                const existing = this.#scope.get(url.hostname) || [];
+                if (!existing.some((u) => u.href === url.href)) {
+                    this.#scope.set(url.hostname, [...existing, url]);
+                }
             }
             this.#linkList.add(url);
         }
@@ -170,7 +244,12 @@ class Crawler extends EventEmitter {
             this.#options.recursive = false;
             this.#options.fromList = true;
         }
-        const isResuming = this.#resumedScraped.length > 0;
+        // A resume can have an empty scraped set — e.g. a crawl interrupted before
+        // any page finished, or a `--retry-failed` run where every page in the
+        // archive was a failure and got reset to pending. Keying purely on
+        // `#resumedScraped` would then mistake the session for a fresh crawl and
+        // drop every resumed pending URL, so honour the pending set too.
+        const isResuming = this.#resumedScraped.length > 0 || this.#resumedPending.length > 0;
         // Dedupe by the same protocol-agnostic key the dealer uses internally.
         // Append-mode in particular can put the same URL into both
         // `#resumedPending` (via `repromoteExternalPages`) and `urls` (the
@@ -185,18 +264,43 @@ class Crawler extends EventEmitter {
             seenInitial.add(key);
             initialUrls.push(url);
         }
-        const resumeOffset = this.#resumedScraped.length;
-        if (initialUrls.length === 0) {
-            crawlerLog('Crawl End (nothing to resume)');
-            void this.emit('crawlEnd', {});
-            return;
+        const root = initialUrls[0];
+        if (!root) {
+            if (isResuming) {
+                crawlerLog('Crawl End (nothing to resume)');
+                void this.emit('crawlEnd', {});
+                return;
+            }
+            throw new Error('urls is empty');
         }
-        void this.#runDeal(initialUrls, resumeOffset).catch((error) => {
+        const resumeOffset = this.#resumedScraped.length;
+        const pagesScrapedOffset = this.#resumedPagesScraped;
+        void this.#runDeal(initialUrls, resumeOffset, pagesScrapedOffset).catch((error) => {
             crawlerLog('runDeal error: %O', error);
             this.#emitDealErrors(error, root.href);
             void this.emit('crawlEnd', {});
         });
     }
+    /**
+     * Thin instance-bound adapter over {@link drainPhaseErrors}. Flushes
+     * `#pendingPhaseErrors` for `url` as `pageError` events. Idempotent.
+     *
+     * **Test gap (known)**: this adapter is invoked from the worker body in
+     * {@link Crawler.#runDeal} at three call sites — after `#handleResult`,
+     * inside the worker's `catch`, and via `logUndrainedPhaseErrors` in
+     * `finally`. The drain logic itself is unit-tested in
+     * `drain-phase-errors.spec.ts`; the wiring (whether the worker actually
+     * calls it on each path) is verified by code review only, because
+     * driving the worker requires a Puppeteer + beholder mock stack whose
+     * cost outweighs the regression it would catch.
+     * @param url - URL whose buffered errors should be flushed.
+     * @param isExternal - Whether the URL is external to the crawl scope.
+     */
+    #drainPhaseErrors(url, isExternal) {
+        drainPhaseErrors(this.#pendingPhaseErrors, url.href, isExternal, (payload) => {
+            void this.emit('pageError', payload);
+        });
+    }
     /**
      * Emits error events for a deal-level failure.
      *
@@ -222,19 +326,22 @@ class Crawler extends EventEmitter {
      * Processes captured sub-resources from a page scrape, deduplicates them,
      * and emits `response` / `responseReferrers` events for new resources.
      * @param resources - Sub-resource entries captured during the page load
+     * @param parentSource
      */
-    #handleResources(resources) {
-        for (const { resource, pageUrl } of resources) {
-            const { isNew } = handleResourceResponse(resource, this.#resources);
-            if (isNew) {
-                void this.emit('response', {
-                    resource: resource,
-                });
-            }
-            void this.emit('responseReferrers', {
-                url: pageUrl,
-                src: resource.url.withoutHash,
-            });
+    #handleResources(resources, parentSource) {
+        // Decide the full emit plan first via the pure planner — that lets
+        // the lineage propagation contract (parent source → sub-resource
+        // `source`) be unit-tested in `plan-sub-resource-emits.spec.ts`
+        // without spinning up the puppeteer stack here. The previous
+        // inline shape made the `source` value invisible to tests because
+        // emit() side effects were only observable via a full scrape run
+        // that requires a mocked Chromium instance.
+        const { responseEmits, referrerEmits } = planSubResourceEmits(resources, parentSource, this.#resources);
+        for (const payload of responseEmits) {
+            void this.emit('response', payload);
+        }
+        for (const payload of referrerEmits) {
+            void this.emit('responseReferrers', payload);
         }
     }
     /**
@@ -246,20 +353,22 @@ class Crawler extends EventEmitter {
      * - `error`: Creates a fallback PageData, marks as done, and emits `error`.
      * @param result - The scrape result from beholder
      * @param url - The URL that was scraped
-     * @param push - Dealer's push callback to enqueue newly discovered URLs
+     * @param enqueue - Callback to enqueue newly discovered URLs into the dealer
+     *   queue, prioritising likely-HTML URLs to the front (see {@link partitionUrlsByHtml}).
+     *   Accepts a batch so a group of URLs (e.g. predicted pagination) keeps its order.
      * @param paginationState - Mutable state for predicted pagination cascade prevention
      * @param paginationState.lastPushedUrl
      * @param paginationState.lastPushedWasPredicted
      * @param concurrency - Current concurrency level, used to determine predicted URL count
      */
-    #handleResult(result, url, push, paginationState, concurrency) {
+    #handleResult(result, url, enqueue, paginationState, concurrency) {
         switch (result.type) {
             case 'success': {
                 if (!result.pageData)
                     break;
                 handleScrapeEnd(result.pageData, this.#linkList, this.#scope, this.#options, (newUrl, opts) => {
                     this.#linkList.add(newUrl, opts);
-                    void push(newUrl);
+                    void enqueue(newUrl);
                     // Predicted pagination detection
                     if (!paginationState || !concurrency)
                         return;
@@ -276,13 +385,18 @@ class Crawler extends EventEmitter {
                         const pattern = detectPaginationPattern(paginationState.lastPushedUrl, newUrl.withoutHashAndAuth);
                         if (pattern) {
                             const urls = generatePredictedUrls(pattern, newUrl.withoutHashAndAuth, concurrency);
+                            const specUrls = [];
                             for (const specUrlStr of urls) {
                                 const specUrl = parseUrl(specUrlStr, this.#options);
                                 if (specUrl) {
                                     this.#linkList.add(specUrl, { predicted: true });
-                                    void push(specUrl);
+                                    specUrls.push(specUrl);
                                 }
                             }
+                            // Enqueue as one batch so ascending page order is kept
+                            // at the front of the queue (see enqueue in #runDeal).
+                            if (specUrls.length > 0)
+                                void enqueue(...specUrls);
                             paginationState.lastPushedUrl = newUrl.withoutHashAndAuth;
                             paginationState.lastPushedWasPredicted = true;
                             return;
@@ -291,11 +405,20 @@ class Crawler extends EventEmitter {
                     paginationState.lastPushedUrl = newUrl.withoutHashAndAuth;
                     paginationState.lastPushedWasPredicted = false;
                 });
-                if (result.pageData.isExternal) {
-                    void this.emit('externalPage', { result: result.pageData });
-                }
-                else {
-                    void this.emit('page', { result: result.pageData });
+                {
+                    const pageSource = derivePageSource(this.#options.inventoryMode, result.pageData.url.withoutHashAndAuth);
+                    if (result.pageData.isExternal) {
+                        void this.emit('externalPage', {
+                            result: result.pageData,
+                            source: pageSource,
+                        });
+                    }
+                    else {
+                        void this.emit('page', {
+                            result: result.pageData,
+                            source: pageSource,
+                        });
+                    }
                 }
                 break;
             }
@@ -324,11 +447,15 @@ class Crawler extends EventEmitter {
                 }, this.#linkList, this.#scope, this.#options);
                 const isExternal = findScopeEntry(url, this.#scope, this.#options) === null;
                 if (pageResult) {
+                    const pageSource = derivePageSource(this.#options.inventoryMode, pageResult.url.withoutHashAndAuth);
                     if (pageResult.isExternal) {
-                        void this.emit('externalPage', { result: pageResult });
+                        void this.emit('externalPage', {
+                            result: pageResult,
+                            source: pageSource,
+                        });
                     }
                     else {
-                        void this.emit('page', { result: pageResult });
+                        void this.emit('page', { result: pageResult, source: pageSource });
                     }
                 }
                 void this.emit('error', {
@@ -355,68 +482,60 @@ class Crawler extends EventEmitter {
      * @param headCheckResult - Optional HEAD result to pass to the scraper, avoiding a redundant request
      * @returns The scrape result from beholder
      */
-    async #launchBrowserAndScrape(url, update, isExternal, metadataOnly, headCheckResult) {
-        update('Launching browser%dots%');
-        if (this.#options.executablePath) {
-            const execPath = path.resolve(this.#options.executablePath);
-            if (!existsSync(execPath)) {
-                throw new Error(`Executable path does not exist: ${execPath}`);
-            }
+    /**
+     * @param url
+     * @param update
+     * @param isExternal
+     * @param metadataOnly
+     * @param headCheckResult
+     * @internal
+     * cascade-guard contract for the puppeteer-fallback success / skipped
+     * branches can be exercised via `vi.spyOn(Crawler.prototype,
+     * '_launchBrowserAndScrape')` in unit tests. There is no production
+     * consumer outside this class.
+     */
+    /**
+     * Resolve the source label of the page being scraped so sub-resources
+     * captured during its render can inherit the correct lineage label
+     * (`'inventory-discovered'` when the parent is in the inventory chain,
+     * `undefined` otherwise so the DB DEFAULT `'crawled'` lands).
+     *
+     * Two-stage resolution:
+     *
+     * 1. If `inventoryMode` is active (live `--inventory` session), use
+     *    `derivePageSource` directly — the in-memory seed set is the
+     *    authoritative answer and no DB round-trip is needed.
+     *
+     * 2. Otherwise (`--resume`, `--retry-failed`, `--append`, or a normal
+     *    `crawl` of a previously-inventoried archive), ask the injected
+     *    `lookupPageSource` callback. The orchestrator wires that callback
+     *    to `Archive.getPageSourceByUrl` so the parent's lineage from
+     *    earlier sessions survives across sessions.
+     *
+     * One round-trip per page render at most — the result is not memoised
+     * because each worker scrapes a single page per `#scrapePage` call
+     * and the cost is amortised across every sub-resource of that page.
+     * @param url - The URL of the page being scraped.
+     * @returns The parent page's source, or `undefined` when none applies.
+     */
+    async #resolveParentSource(url) {
+        const fromInventoryMode = derivePageSource(this.#options.inventoryMode, url.withoutHashAndAuth);
+        if (fromInventoryMode !== undefined) {
+            return fromInventoryMode;
+        }
+        const lookupPageSource = this.#options.lookupPageSource;
+        if (!lookupPageSource) {
+            return undefined;
         }
-        const puppeteer = await import('puppeteer');
-        const browser = await puppeteer.launch({
-            headless: true,
-            ...(this.#options.executablePath
-                ? { executablePath: this.#options.executablePath }
-                : {}),
-        });
         try {
-            update('Creating page%dots%');
-            const page = await browser.newPage();
-            await page.setUserAgent(this.#options.userAgent);
-            // Defence-in-depth: beholder sets Authorization via setExtraHTTPHeaders,
-            // but page.authenticate() handles Chromium-level HTTP auth challenges
-            // (401 + WWW-Authenticate) that setExtraHTTPHeaders cannot cover.
-            if (url.username && url.password) {
-                await page.authenticate({
-                    username: url.username,
-                    password: url.password,
-                });
-            }
-            const scraper = new Scraper();
-            scraper.on('changePhase', (e) => {
-                const msg = formatPhaseLog(e);
-                if (msg) {
-                    update(msg);
-                }
-                void this.emit('changePhase', e);
-            });
-            const result = await scraper.scrapeStart(page, url, {
-                isExternal,
-                captureImages: !isExternal && this.#options.captureImages,
-                excludeKeywords: this.#options.excludeKeywords,
-                disableQueries: this.#options.disableQueries,
-                metadataOnly,
-                retries: this.#options.retry,
-                headCheckResult,
-            });
-            update('Closing browser%dots%');
-            return result;
+            return await lookupPageSource(url.withoutHashAndAuth);
         }
         catch (error) {
-            return {
-                type: 'error',
-                resources: [],
-                error: {
-                    name: error instanceof Error ? error.name : 'Error',
-                    message: error instanceof Error ? error.message : String(error),
-                    stack: error instanceof Error ? error.stack : undefined,
-                    shutdown: true,
-                },
-            };
-        }
-        finally {
-            await browser.close().catch(() => { });
+            // A lookup failure must never be worse than not having lineage
+            // — fall back to undefined so the sub-resources land at the DB
+            // DEFAULT `'crawled'` rather than crashing the whole worker.
+            crawlerLog('Parent source lookup failed for %s: %O', url.href, error);
+            return undefined;
         }
     }
     /**
@@ -429,16 +548,29 @@ class Crawler extends EventEmitter {
      * @param initialUrls - Starting URLs to seed the deal queue
      * @param resumeOffset - Number of URLs already scraped in a previous session,
      *   added to the progress counter for accurate display
+     * @param pagesScrapedOffset - Number of HTML pages already rendered in previous
+     *   sessions, used to seed the per-session HTML-pages counter so the display
+     *   remains accurate across resumes
      */
-    async #runDeal(initialUrls, resumeOffset = 0) {
+    async #runDeal(initialUrls, resumeOffset = 0, pagesScrapedOffset = 0) {
         const seen = new Set(initialUrls.map((u) => protocolAgnosticKey(u.withoutHashAndAuth)));
         // Add scraped URLs to seen to prevent re-processing during resume
         for (const url of this.#resumedScraped) {
             seen.add(protocolAgnosticKey(url));
         }
+        // Redirect-destination dedup is per-crawl; clear any state from a prior run.
+        this.#scrapedDestinations.clear();
+        // Session-liveness signal is per-crawl too; clear so a fresh session
+        // does not inherit "host alive" claims from a prior run that may have
+        // happened on an entirely different network.
+        this.#successfulHosts.clear();
         // external URL の追跡（target は deal の total/done から導出）
         const externalUrls = new Set();
         const externalDoneUrls = new Set();
+        // HTML ページとしてブラウザでレンダリングし、かつアーカイブに保存されたページ数。
+        // HEAD のみ・title 取得のみ・skip・ブラウザ起動失敗・predicted-discard は含まない。
+        // 過去セッションぶんは pagesScrapedOffset として init される。
+        let pagesScraped = pagesScrapedOffset;
         // 初期 URL を分類（onPush を通らないため）
         for (const url of initialUrls) {
             if (findScopeEntry(url, this.#scope, this.#options) === null) {
@@ -453,7 +585,7 @@ class Crawler extends EventEmitter {
             lastPushedUrl: null,
             lastPushedWasPredicted: false,
         };
-        await deal(initialUrls, (url, update, _index, setLineHeader, push) => {
+        await deal(initialUrls, (url, update, _index, setLineHeader, push, unshift) => {
             const matchedScope = findScopeEntry(url, this.#scope, this.#options);
             const isExternal = matchedScope === null;
             const urlText = isExternal ? c.dim(url.href) : c.cyan(url.href);
@@ -463,8 +595,44 @@ class Crawler extends EventEmitter {
             }
             this.#linkList.add(url);
             this.#linkList.progress(url);
+            // Likely-HTML URLs jump to the front of the queue (unshift) so page
+            // crawling advances ahead of asset/document fetches; everything else
+            // is appended (push). partitionUrlsByHtml splits the batch by the
+            // URL-only heuristic. Variadic so a batch (e.g. predicted pagination)
+            // keeps its order: a single unshift(...html) preserves ascending order
+            // at the front, whereas unshifting one-by-one would reverse it.
+            const enqueue = (...newUrls) => {
+                const [html, other] = partitionUrlsByHtml(newUrls);
+                const ops = [];
+                if (html.length > 0)
+                    ops.push(unshift(...html));
+                if (other.length > 0)
+                    ops.push(push(...other));
+                return Promise.all(ops).then(() => { });
+            };
             return async () => {
+                // Interval delay is handled here instead of by dealer because
+                // DNS-burned hosts must skip the wait entirely. Spending the
+                // per-URL interval on a host the cache already knows is dead
+                // just slows the crawl down for zero benefit — the HEAD won't
+                // be fired and `Crawler.#sendHeadRequest` will throw the
+                // preload short-circuit immediately. For all other URLs, run
+                // the same `delay()` + `%countdown(...)` log that dealer would
+                // have emitted, so the dealer display reads identically.
+                const burned = dnsBurnedHostCache.has(url.hostname.toLowerCase());
+                if (!burned && this.#options.interval && this.#options.interval > 0) {
+                    await delay(this.#options.interval, (determinedInterval) => {
+                        update(`Waiting interval: %countdown(${determinedInterval},${_index}_interval)%ms`);
+                    });
+                }
                 const log = createTimedUpdate(update, this.#options.verbose);
+                // `#scrapePage` 内のブラウザ HTML レンダーが成功したかをマークするフラグ。
+                // 成功時のみ #scrapePage 側で true に設定される。
+                // discard 判定後にこのフラグを見てカウントするので、launch 失敗や predicted-discard は除外される。
+                let renderedInBrowser = false;
+                const markBrowserScrape = () => {
+                    renderedInBrowser = true;
+                };
                 try {
                     const robotsAllowed = await this.#robotsChecker.isAllowed(url);
                     if (!robotsAllowed) {
@@ -496,24 +664,112 @@ class Crawler extends EventEmitter {
                             isLowerLayer: false,
                         });
                         this.#linkList.done(url, this.#scope, { page: pageData }, this.#options);
-                        void this.emit('externalPage', { result: pageData });
+                        void this.emit('externalPage', {
+                            result: pageData,
+                            source: derivePageSource(this.#options.inventoryMode, url.withoutHashAndAuth),
+                        });
                         log(c.dim('External (skip fetch)'));
                         return;
                     }
                     const metadataOnly = this.#linkList.isMetadataOnly(url.withoutHash);
                     const isPredicted = this.#linkList.isPredicted(url.withoutHashAndAuth);
                     log('Scraping%dots%');
-                    const result = await this.#scrapePage(url, log, metadataOnly, _index);
+                    const result = await this.#scrapePage(url, log, metadataOnly, _index, markBrowserScrape);
+                    // Redirect convergence (#73): the destination was already
+                    // rendered during this crawl, so only the redirect edge is
+                    // recorded and the browser was never launched. Mark the URL
+                    // done and emit `redirect` (routed to `Archive.setRedirect`,
+                    // which writes the edge without touching the destination's
+                    // content). This URL does not count toward pagesScraped.
+                    if (result.type === 'redirect-edge') {
+                        // Note: a predicted (speculative) URL that reaches here genuinely
+                        // redirects (the server returned 3xx), so it is a real URL — we
+                        // record its edge rather than discard it. This matches the render
+                        // path, where the first predicted source to a destination renders
+                        // it and is recorded as a redirect source the same way; only 404 /
+                        // error predicted URLs are dropped (by `shouldDiscardPredicted`).
+                        //
+                        // The `source` discriminator divides this branch in two:
+                        //
+                        // - `'http-chain'` — the HEAD pre-flight resolved a real 3xx chain
+                        //   and the destination is already rendered (`#scrapedDestinations`
+                        //   claim). Every URL in `redirectPaths` is intermediate / known,
+                        //   so the existing behaviour applies: `linkList.done` folds the
+                        //   whole chain into the done-set so later references skip cleanly.
+                        //
+                        // - `'js-redirect'` — `scraper.scrapeStart` threw because
+                        //   `page.goto()` returned null (`window.location.replace()` /
+                        //   meta-refresh fired mid-navigation), and `redirectPaths`
+                        //   carries the single JS target Chromium ended up on. That target
+                        //   has NOT been rendered yet — it must enter the crawl queue, and
+                        //   `linkList.done` MUST NOT fold it into the done-set (otherwise
+                        //   the dealer's `seen` rejects the push and the destination is
+                        //   silently lost from the archive).
+                        if (result.source === 'js-redirect') {
+                            const destination = result.pageData.redirectPaths.at(-1);
+                            if (destination) {
+                                const destinationUrl = parseUrl(destination, this.#options);
+                                if (destinationUrl) {
+                                    this.#linkList.add(destinationUrl);
+                                    void enqueue(destinationUrl);
+                                }
+                                else {
+                                    // `deriveJsRedirectTarget` already canonicalises
+                                    // via WHATWG URL parsing, so reaching the
+                                    // `parseUrl === null` branch here would mean
+                                    // `@d-zero/shared/parse-url` rejected what
+                                    // WHATWG accepted — unexpected, and silently
+                                    // dropping the destination would be a silent
+                                    // archive loss. Log it so DEBUG=Nitpicker:Crawler
+                                    // catches the case.
+                                    crawlerLog('JS-redirect destination %s failed to parse — dropping enqueue', destination);
+                                }
+                            }
+                            else {
+                                crawlerLog('JS-redirect result for %s had no redirectPaths destination — dropping enqueue', url.href);
+                            }
+                            this.#linkList.done(url, this.#scope, { page: result.pageData }, this.#options, { includeRedirectPaths: false });
+                        }
+                        else {
+                            this.#linkList.done(url, this.#scope, { page: result.pageData }, this.#options);
+                        }
+                        // The redirect-edge call path may INSERT a brand-new
+                        // destination row (js-redirect rescue, #73
+                        // convergence on first sight). Forward the
+                        // originating page's inventory provenance so the
+                        // destination + intermediate hops inherit the
+                        // chain's lineage instead of laundering to DB
+                        // DEFAULT `'crawled'`. `inventoryMode === null`
+                        // (resume / retry-failed) yields `undefined`,
+                        // which is correct: the DB-side lookup in
+                        // `#linkRedirectSources` reads the destination's
+                        // stored source for those sessions.
+                        void this.emit('redirect', buildRedirectEvent(result.pageData, this.#options.inventoryMode, url.withoutHashAndAuth));
+                        log(c.dim('Redirect (dest already scraped)'));
+                        return;
+                    }
                     // Discard predicted URLs that failed (404, error, etc.)
                     if (isPredicted && shouldDiscardPredicted(result)) {
                         handleIgnoreAndSkip(url, this.#linkList, this.#scope, this.#options);
                         log(c.dim('Predicted (discarded)'));
                         return;
                     }
+                    // Count only after discard check: rendered HTML pages that
+                    // will be persisted to the archive. Launch failures bypass
+                    // this point via the catch block; discarded predicted URLs
+                    // return above without reaching here.
+                    if (renderedInBrowser) {
+                        pagesScraped++;
+                    }
                     log('Saving results%dots%');
-                    this.#handleResult(result, url, push, paginationState, concurrency);
-                    this.#handleResources(result.resources);
+                    this.#handleResult(result, url, enqueue, paginationState, concurrency);
+                    const parentSource = await this.#resolveParentSource(url);
+                    this.#handleResources(result.resources, parentSource);
                     log(formatResultSummary(result));
+                    // Phase errors must be emitted AFTER 'page' / 'externalPage'
+                    // so the orchestrator's WriteQueue sees `setPage` before
+                    // `insertPageError` and the URL→pageId resolution succeeds.
+                    this.#drainPhaseErrors(url, isExternal);
                 }
                 catch (error) {
                     crawlerLog('Worker error for %s: %O', url.href, error);
@@ -532,16 +788,29 @@ class Crawler extends EventEmitter {
                         isExternal,
                         error: workerError,
                     });
+                    // Hard-error path: persist whatever phase errors we have
+                    // already buffered so they are not lost.
+                    this.#drainPhaseErrors(url, isExternal);
                 }
                 finally {
                     if (isExternal) {
                         externalDoneUrls.add(protocolAgnosticKey(url.withoutHashAndAuth));
                     }
+                    // Phase errors still in the buffer here were not drained
+                    // by the success or catch paths — typically because a
+                    // predicted URL was discarded before reaching the drain
+                    // point. The helper logs the drop (observable via
+                    // DEBUG=Nitpicker:Crawler) and removes the entry so the
+                    // Map cannot leak across crawls.
+                    logUndrainedPhaseErrors(this.#pendingPhaseErrors, url.href, crawlerLog);
                 }
             };
         }, {
             limit: concurrency,
-            interval: this.#options.interval,
+            // Interval is applied per-URL inside the worker callback above so
+            // DNS-burned hosts can skip it. Letting dealer handle interval
+            // would run the wait before our short-circuit check fires.
+            interval: 0,
             verbose: this.#options.verbose || !process.stdout.isTTY,
             signal: this.#abortController.signal,
             header: (_progress, done, total, limit) => {
@@ -551,6 +820,7 @@ class Crawler extends EventEmitter {
                     resumeOffset,
                     externalTotal: externalUrls.size,
                     externalDone: externalDoneUrls.size,
+                    pagesScraped,
                     limit,
                 });
             },
@@ -581,13 +851,55 @@ class Crawler extends EventEmitter {
      * @param update - Callback for progress messages
      * @param metadataOnly - When true, only extract title metadata without full browser scraping
      * @param laneIndex - The dealer lane index, used to create unique countdown IDs
+     * @param markBrowserScrape - Called once **after** the browser successfully
+     *   renders an HTML page (i.e. `_launchBrowserAndScrape` resolved with
+     *   `type: 'success'`). Not called for HEAD-only, title-only, captured-resource
+     *   reuse, non-HTML responses, non-HTTP protocols (mailto:, tel:), browser
+     *   launch throws (e.g. invalid executablePath), or scraper-returned
+     *   `type: 'error'` results. The caller is responsible for further filtering
+     *   (e.g. predicted-discard).
      * @returns The scrape result
      */
-    async #scrapePage(url, update, metadataOnly, laneIndex) {
+    async #scrapePage(url, update, metadataOnly, laneIndex, markBrowserScrape) {
         const isExternal = findScopeEntry(url, this.#scope, this.#options) === null;
         // Non-HTTP protocols (mailto:, tel:, etc.) — let the scraper handle early return
         if (!url.isHTTP) {
-            return this.#launchBrowserAndScrape(url, update, isExternal, metadataOnly);
+            return this._launchBrowserAndScrape(url, update, isExternal, metadataOnly);
+        }
+        // Reuse captured resource data — when this URL was already observed as a
+        // sub-resource during page rendering, its response data is recorded and
+        // the HEAD pre-flight is redundant. Only 2xx non-HTML rows are eligible
+        // (see resourceToPageData); anything else falls through to the pre-flight.
+        // Both URL variants are checked because scope-auth injection adds
+        // credentials to queued URLs while browser-captured resource URLs have none.
+        // The result is deliberately NOT written to destinationCache: a queued URL
+        // is processed at most once (the dealer dedupes by protocol-agnostic key),
+        // so a URL that takes this path never reaches fetchDestination again.
+        const lookupResource = this.#options.lookupResource;
+        if (lookupResource &&
+            (this.#resources.has(url.withoutHash) ||
+                this.#resources.has(url.withoutHashAndAuth))) {
+            update('Checking captured resource%dots%');
+            let resource = null;
+            try {
+                resource = await lookupResource([url.withoutHash, url.withoutHashAndAuth]);
+            }
+            catch (error) {
+                // A lookup failure must never be worse than not having the
+                // optimization — fall back to the HEAD pre-flight below.
+                crawlerLog('Resource lookup failed for %s, falling back: %O', url.href, error);
+            }
+            const pageData = resource
+                ? resourceToPageData({ url, isExternal, resource })
+                : null;
+            if (pageData) {
+                crawlerLog('Reused captured resource for %s', url.href);
+                return {
+                    type: 'success',
+                    pageData: metadataOnly ? { ...pageData, isTarget: false } : pageData,
+                    resources: [],
+                };
+            }
         }
         // Pre-flight: lightweight HEAD request to check server availability
         update('HEAD request%dots%');
@@ -596,6 +908,131 @@ class Crawler extends EventEmitter {
             headCheckResult = await this.#sendHeadRequest(url, isExternal, update, laneIndex);
         }
         catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            // Puppeteer-only fallback: when the HEAD pre-flight (and its GET
+            // companion inside `fetchDestination`) exhaust retries on what
+            // looks like an HTML URL, give the browser exactly one chance
+            // before recording the page as `status = -1`. Some middleboxes /
+            // WAF configurations drop bare HEAD/GET probes (parse-error,
+            // reset, silent timeout) while still answering a real puppeteer
+            // navigation; those URLs would otherwise be permanently lost.
+            //
+            // Restricted to non-metadataOnly scrapes because metadata-only
+            // mode is a bandwidth-saving path for external pages — there is
+            // no payoff in spinning up puppeteer when the row was never
+            // going to be fully rendered. `isPuppeteerFallbackCandidate`
+            // filters PreloadShortCircuitError automatically via its
+            // classifier check (its synthesised message classifies as `dns`).
+            if (!metadataOnly &&
+                isLikelyHtmlUrl(url) &&
+                isPuppeteerFallbackCandidate(errorMessage)) {
+                update(c.yellow('HEAD/GET unreachable — trying puppeteer once'));
+                try {
+                    const fallback = await this._launchBrowserAndScrape(url, update, isExternal, metadataOnly);
+                    if (fallback.type === 'success') {
+                        if (fallback.pageData) {
+                            const renderedKey = redirectDestKey(url, fallback.pageData.redirectPaths);
+                            this.#scrapedDestinations.add(renderedKey);
+                        }
+                        // Puppeteer fallback proved the host is reachable
+                        // (HEAD/GET probes died at a middlebox / WAF but the
+                        // real browser navigation got a response). Mark the
+                        // host alive for the cascade guard — without this, a
+                        // host whose first URL only succeeded via the
+                        // browser-rescue path would still be vulnerable to
+                        // the next URL's HEAD failure burning it.
+                        this.#successfulHosts.add(url.hostname.toLowerCase());
+                        markBrowserScrape();
+                        return fallback;
+                    }
+                    if (fallback.type === 'skipped') {
+                        // Puppeteer rendered the page far enough for the scraper
+                        // to match an `excludeKeywords` rule. That is a definitive
+                        // "skip" verdict from the browser, NOT an unreachable
+                        // host — surface the skip so downstream handling (skip
+                        // counter, anchor-extraction suppression, `setSkippedPage`
+                        // in the archive) behaves identically to the case where
+                        // HEAD had succeeded. Without this branch, the page would
+                        // be recorded as `status = -1` with the HEAD timeout
+                        // message — a misleading entry that conflates
+                        // "operator-intended skip" with "network failure".
+                        //
+                        // Skipped also counts as proof-of-life: the browser
+                        // reached the page far enough to match exclude rules,
+                        // so the host was clearly responding.
+                        this.#successfulHosts.add(url.hostname.toLowerCase());
+                        return fallback;
+                    }
+                    // `fallback.type === 'error'`. `_launchBrowserAndScrape`
+                    // catches its own exceptions and returns
+                    // `{type:'error', shutdown:...}` rather than throwing, so
+                    // the `catch` arm below would NOT see this branch. Log
+                    // the puppeteer-side cause (and any `shutdown` flag the
+                    // scraper attached) so operators have a breadcrumb that
+                    // the safety net actually fired and lost — otherwise
+                    // only the HEAD error reaches `crawl_errors` and the
+                    // browser failure mode is invisible.
+                    crawlerLog('Puppeteer fallback returned error for %s: %s (shutdown=%s)', url.href, fallback.error?.message ?? '(no message)', fallback.error?.shutdown ?? false);
+                    // JS-redirect rescue on the puppeteer-fallback branch:
+                    // the HEAD/GET probes died (the kind set in
+                    // `isPuppeteerFallbackCandidate` — middlebox / WAF
+                    // shapes), the one-shot puppeteer attempt also threw,
+                    // but `page.url()` reported a different post-navigation
+                    // URL. This is the same WAF-+-JS-redirect shape the
+                    // HEAD-success rescue handles one branch below, applied
+                    // to the prior failure layer. Without this, a URL whose
+                    // only sin is "HEAD blocked + JS-redirected body" falls
+                    // to `status = -1` and joins the retry-forever loop the
+                    // rescue is supposed to break. The trigger is the same
+                    // narrow `Page.goto returned null` shape — anything
+                    // else (TLS failure inside puppeteer, target crash, …)
+                    // must fall through to the unreachable path so the real
+                    // failure surfaces. We synthesise the redirect-edge
+                    // PageData from the HEAD error (status = -1) instead of
+                    // from a HEAD success, so `#linkRedirectSources` still
+                    // stamps the source as 301 and the edge wires the dest
+                    // in.
+                    const fallbackRescue = buildJsRedirectEdge({
+                        url,
+                        isExternal,
+                        errorMessage: fallback.error?.message,
+                        postNavigationUrl: fallback.postNavigationUrl,
+                        // No `headCheckResult`: HEAD itself died on this
+                        // path, so the synthesised PageData starts from a
+                        // `linkToPageData` placeholder with `status = -1`
+                        // carrying the original HEAD error message.
+                        // `#linkRedirectSources` still flips the source row
+                        // to 301 because NULL/-1 satisfies its conditional
+                        // stamp predicate.
+                    });
+                    if (fallbackRescue !== null) {
+                        return fallbackRescue;
+                    }
+                }
+                catch (browserError) {
+                    // Browser launch / runtime crash — fall through to the
+                    // unreachable path below. The original HEAD error is more
+                    // informative about WHY the URL wasn't reachable, so it
+                    // (not the puppeteer noise) is what we surface in
+                    // `crawl_errors`. The lane display flag below (
+                    // "Unreachable (fallback failed)") preserves the fact
+                    // that puppeteer also tried, so operators reading the
+                    // progress log can tell this URL got the safety-net
+                    // attempt versus the cheap-probe-only path.
+                    crawlerLog('Puppeteer fallback also failed for %s: %O', url.href, browserError);
+                }
+                update(c.red('Unreachable (fallback failed)'));
+                return {
+                    type: 'error',
+                    resources: [],
+                    error: {
+                        name: error instanceof Error ? error.name : 'Error',
+                        message: errorMessage,
+                        stack: error instanceof Error ? error.stack : undefined,
+                        shutdown: false,
+                    },
+                };
+            }
             // Server unreachable — skip browser launch entirely
             update(c.red('Unreachable'));
             return {
@@ -603,16 +1040,38 @@ class Crawler extends EventEmitter {
                 resources: [],
                 error: {
                     name: error instanceof Error ? error.name : 'Error',
-                    message: error instanceof Error ? error.message : String(error),
+                    message: errorMessage,
                     stack: error instanceof Error ? error.stack : undefined,
                     shutdown: false,
                 },
             };
         }
+        // Redirect convergence (#73): `finalKey` is the destination this URL lands
+        // on after following its redirect chain (or the URL itself when it does not
+        // redirect). When that destination has already been rendered and stored
+        // during this crawl, do NOT process this URL further — record the redirect
+        // edge only and skip everything below, regardless of content type. This is
+        // the root fix for the many-to-one redirect duplication (#70): every source
+        // URL that 301s to one destination otherwise re-renders/re-stores it. The
+        // check sits ABOVE the metadata-only and non-HTML branches on purpose — both
+        // route their HEAD/title result through `updatePage`, which would funnel a
+        // content-less result into `#insertPage` and overwrite the already-rendered
+        // destination's title / meta / isExternal. The edge-only path leaves the
+        // destination row intact.
+        //
+        // `finalKey` is also claimed for destinations reached directly (no redirect;
+        // see the claim after a successful render below), so a destination that is
+        // both linked directly and arrived at via a redirect is rendered by whichever
+        // path wins the race, not both.
+        const finalKey = redirectDestKey(url, headCheckResult.redirectPaths);
+        if (this.#scrapedDestinations.has(finalKey)) {
+            crawlerLog('Redirect dest already rendered, edge only: %s', url.href);
+            return { type: 'redirect-edge', source: 'http-chain', pageData: headCheckResult };
+        }
         // Title-only mode — extract <title> via partial GET for HTML, skip browser
         if (metadataOnly) {
             if (headCheckResult.contentType === null ||
-                headCheckResult.contentType === 'text/html') {
+                isHtmlContentType(headCheckResult.contentType)) {
                 update('Fetching title%dots%');
                 try {
                     const titleResult = await fetchDestination({
@@ -640,15 +1099,135 @@ class Crawler extends EventEmitter {
         }
         // Non-HTML content — skip browser
         if (headCheckResult.contentType !== null &&
-            headCheckResult.contentType !== 'text/html') {
+            !isHtmlContentType(headCheckResult.contentType)) {
             return {
                 type: 'success',
                 pageData: headCheckResult,
                 resources: [],
             };
         }
-        // HTML or unknown content type — launch browser with preflight result
-        return this.#launchBrowserAndScrape(url, update, isExternal, metadataOnly, headCheckResult);
+        // HTML or unknown content type — launch browser with preflight result.
+        // markBrowserScrape() fires only when the result is `success`.
+        // `_launchBrowserAndScrape` catches internal errors and returns
+        // `{ type: 'error', ... }` instead of throwing (see its catch block),
+        // so awaiting alone does NOT prove the page was rendered. The explicit
+        // success check excludes navigation failures, scraper exceptions, and
+        // shutdown-class errors from the pages-rendered count.
+        const browserResult = await this._launchBrowserAndScrape(url, update, isExternal, metadataOnly, headCheckResult);
+        if (browserResult.type === 'success') {
+            markBrowserScrape();
+            // Claim the destination that was ACTUALLY rendered, keyed off the
+            // browser's own redirect resolution rather than the HEAD pre-flight's
+            // guess (`finalKey`). The browser is authoritative for what got stored;
+            // if HEAD and the browser disagree on the final URL (method-conditional
+            // / JS / meta-refresh redirects), keying the claim off the HEAD guess
+            // would route a sibling source to an edge pointing at a never-rendered
+            // phantom row. By claiming the rendered URL, a divergent sibling simply
+            // re-renders (dedup misses) instead — correct, just less optimal. In the
+            // common case HEAD and the browser agree, so the keys are identical.
+            //
+            // Claimed only after a successful render, so a failed render leaves the
+            // destination unclaimed and a later source retries it. Concurrent
+            // in-flight sources to the same destination (bounded by the concurrency
+            // limit) may still each render before any claim lands; the storage-layer
+            // replace in `updatePage` (#70) keeps the resulting anchors / images
+            // correct (sub-resources may briefly duplicate, far below the pre-#73
+            // once-per-source blow-up).
+            const renderedKey = browserResult.pageData
+                ? redirectDestKey(url, browserResult.pageData.redirectPaths)
+                : finalKey;
+            this.#scrapedDestinations.add(renderedKey);
+            return browserResult;
+        }
+        // Browser scrape failed but the HEAD pre-flight already resolved a
+        // redirect chain — fall back to the redirect-edge path so the chain
+        // is not lost. Without this, a URL whose final destination is on
+        // HTTPS→HTTP downgrade (or any other navigation Chromium refuses
+        // to complete while the underlying redirect was a normal 301/302)
+        // would be persisted as `status = -1` with NULL `redirectDestId`,
+        // then re-picked up by every `--retry-failed` pass forever — the
+        // HEAD answer is the authoritative truth and the browser cannot
+        // invalidate it.
+        //
+        // Restricted to `type === 'error'` because:
+        // - `'skipped'` is an `excludeKeywords` verdict from the browser
+        //   on the rendered URL and is its own definitive outcome —
+        //   surfacing it as a redirect-edge would lose the skip signal.
+        // - `'success'` is handled above.
+        //
+        // The destination is claimed even though no row was rendered for
+        // it: subsequent siblings on the same chain should also fold into
+        // the same edge instead of re-firing the same failing browser
+        // attempt. If the destination URL itself reaches the queue later,
+        // it goes through the normal `#scrapePage` path (the claim only
+        // short-circuits sibling redirect SOURCES, not the destination
+        // itself).
+        if (browserResult.type === 'error' && headCheckResult.redirectPaths.length > 0) {
+            this.#scrapedDestinations.add(finalKey);
+            crawlerLog('Browser scrape failed for %s but HEAD resolved a redirect chain — recording as edge', url.href);
+            return { type: 'redirect-edge', source: 'http-chain', pageData: headCheckResult };
+        }
+        // JS-redirect rescue: HEAD returned a definitive response (no chain),
+        // the browser scrape threw with the specific `Page.goto returned null`
+        // shape (gated by `isJsRedirectErrorShape` below), and puppeteer
+        // reports a different post-navigation URL via `page.url()`. The
+        // motivating case is a server returning `200 OK` whose body contains
+        // `window.location.replace(...)` or `<meta http-equiv="refresh">` —
+        // `page.goto()` resolves to `null` once the JS-driven navigation
+        // supersedes the original, and the scraper throws
+        // `The method Page.goto returned null`. Recording the edge preserves
+        // the link from the source to the JS-redirect target, removes the
+        // page from `--retry-failed`'s candidate pool (the SQL filter
+        // excludes rows with a non-null `redirectDestId`), and matches what
+        // a real browser shows the user.
+        //
+        // What the source row reads as:
+        // - the source is not committed via `setPage`/`updatePage` on this
+        //   path (the redirect-edge handler in `#runDeal` only calls
+        //   `linkList.done` + `emit('redirect', ...)` → `Archive.setRedirect`),
+        //   so `recordRedirect` → `#getIdByUrl` creates a NULL-status
+        //   placeholder row for the source if it did not already exist;
+        // - `#linkRedirectSources` then stamps `status = 301
+        //   statusText='Moved Permanently'` because NULL satisfies its
+        //   conditional-update predicate.
+        // That is the same shape an HTTP 301 source ends up with — the
+        // truthful HTTP layer (the upstream's 200) is lost on this path, but
+        // the alternative (status=-1 retry-forever) is strictly worse. A
+        // future refinement could keep the HEAD-derived status by routing
+        // the source through `setPage` before `setRedirect`; intentionally
+        // deferred to keep this rescue minimal.
+        //
+        // Pre-claiming the destination in `#scrapedDestinations` would
+        // short-circuit the freshly-enqueued destination at the top of
+        // `#scrapePage` (the `if (#scrapedDestinations.has(finalKey))` guard
+        // at line 1213), leaving the dest row as a content-less HEAD edge
+        // instead of a fully rendered page. So we *do not* claim here — the
+        // destination renders normally via the queue, and `#scrapedDestinations`
+        // is populated at line ~1322 of the render-success path the way every
+        // other URL is. Sibling JS-redirect sources to the same destination
+        // still converge: the second sibling enters this branch, observes its
+        // own `page.url()` landing on the same target, records its own
+        // redirect-edge, and re-enqueues — the dealer's `seen` dedup absorbs
+        // the duplicate push, so the destination renders exactly once.
+        if (browserResult.type === 'error') {
+            const headSuccessRescue = buildJsRedirectEdge({
+                url,
+                isExternal,
+                errorMessage: browserResult.error?.message,
+                postNavigationUrl: browserResult.postNavigationUrl,
+                // `headCheckResult` is supplied here so the synthesised
+                // PageData carries the real HTTP-level status / content
+                // type from the HEAD pre-flight. `#linkRedirectSources`
+                // only stamps 301 onto NULL/-1 status rows, so the
+                // HEAD-derived status DOES survive on this path — the
+                // truthful HTTP 200 is preserved.
+                headCheckResult,
+            });
+            if (headSuccessRescue !== null) {
+                return headSuccessRescue;
+            }
+        }
+        return browserResult;
     }
     /**
      * Performs a pre-flight HTTP HEAD request with retry logic.
@@ -663,17 +1242,244 @@ class Crawler extends EventEmitter {
      * @returns Lightweight page data from the HEAD response
      */
     async #sendHeadRequest(url, isExternal, update, laneIndex) {
-        return retryCall(() => fetchDestination({ url, isExternal, userAgent: this.#options.userAgent }), {
+        const host = url.hostname.toLowerCase();
+        if (dnsBurnedHostCache.has(host)) {
+            // Either session-learned earlier in this crawl (one URL on this host
+            // already exhausted retries with a DNS error) or preload-seeded from
+            // `crawl_errors` on archive open. Either way: skip the HEAD entirely.
+            // The orchestrator's error-channel listener detects
+            // PreloadShortCircuitError via instanceof and refuses to write it to
+            // `crawl_errors`, preventing self-amplification across crawls.
+            dnsBurnedHostShortCircuitCounter.count++;
+            update(c.red(`HEAD request: host ${host} DNS-burned — skipping`));
+            throw new PreloadShortCircuitError(host);
+        }
+        // Escalating per-attempt timeout: a slow-but-reachable server (e.g. some
+        // government sites under load) often answers in 20-40 s but is missed by
+        // a flat 10 s race on every retry. Start short to keep crawl throughput
+        // up on healthy URLs, then back off so the last attempt is generous
+        // enough that "really slow" gets a fair shot before we give up.
+        let attempt = 0;
+        return retryCall(async () => {
+            // Clamp the attempt index to the last entry of the escalation array
+            // so retry counts past the array length keep using the longest
+            // budget instead of falling off into `undefined`. `as number`
+            // only because TS can't see that a positive-length readonly array
+            // always has a defined last element.
+            const escalationIndex = Math.min(attempt, HEAD_TIMEOUT_ESCALATION_MS.length - 1);
+            const timeoutMs = HEAD_TIMEOUT_ESCALATION_MS[escalationIndex];
+            attempt += 1;
+            const headResult = await fetchDestination({
+                url,
+                isExternal,
+                userAgent: this.#options.userAgent,
+                timeout: timeoutMs,
+            });
+            // Mark host alive the MOMENT an HTTP response is observed,
+            // before retryCall's outer resolution settles. A later attempt
+            // (or a sibling worker's onGiveUp) racing this success would
+            // otherwise see an empty `#successfulHosts` and burn the host
+            // — exactly the cascade the guard is here to prevent. Any HTTP
+            // status counts: the guard cares about DNS-and-TCP reachability,
+            // not application-level success, and `fetchDestination` only
+            // resolves when an HTTP response was actually received.
+            this.#successfulHosts.add(host);
+            return headResult;
+        }, {
             retries: this.#options.retry,
             label: 'HEAD request',
             onWait: (determinedInterval, retryCount, label, error) => {
                 update(`${label}: ${error.message} — %countdown(${determinedInterval},fetchHead_${laneIndex}_${retryCount},s)%s (retry #${retryCount + 1})`);
             },
             onGiveUp: (retryCount, error, label) => {
+                // Burn the host so subsequent URLs short-circuit — but ONLY
+                // when this is the first time we've ever seen the host fail
+                // in this session. A host that responded earlier is treated
+                // as transiently unreachable (operator's resolver flipped
+                // mid-crawl etc.), not a dead domain. `shouldBurnHost`
+                // encapsulates this decision so the cascade guard is
+                // independently testable. Also gated to `onGiveUp` rather
+                // than `onWait` so an `EAI_AGAIN` that recovers on retry
+                // doesn't trip the guard prematurely.
+                if (shouldBurnHost({
+                    errorKind: classifyErrorKind(error.message),
+                    host,
+                    successfulHosts: this.#successfulHosts,
+                })) {
+                    dnsBurnedHostCache.set(host, 'dns');
+                }
                 update(c.red(`${label}: gave up after ${retryCount} retries — ${error.message}`));
             },
         });
     }
+    // eslint-disable-next-line no-restricted-syntax -- intentional `private` (vs `#`) so tests can spyOn the prototype to drive the puppeteer-fallback cascade-guard branches without a full browser mock; see JSDoc above.
+    async _launchBrowserAndScrape(url, update, isExternal, metadataOnly, headCheckResult) {
+        update('Launching browser%dots%');
+        if (this.#options.executablePath) {
+            const execPath = path.resolve(this.#options.executablePath);
+            if (!existsSync(execPath)) {
+                throw new Error(`Executable path does not exist: ${execPath}`);
+            }
+        }
+        const puppeteer = await import('puppeteer');
+        const browser = await puppeteer.launch({
+            headless: true,
+            ...(this.#options.executablePath
+                ? { executablePath: this.#options.executablePath }
+                : {}),
+        });
+        // `page` is hoisted out of the try-block so the catch arm can read
+        // `page.url()` for JS-redirect detection. See `BrowserScrapeResult`
+        // JSDoc for the full why; in short, when `scrapeStart` throws because
+        // `page.goto()` returned `null`, the puppeteer page object still
+        // holds the URL Chromium actually navigated to via the offending
+        // `window.location.replace()` / meta-refresh, and that is the only
+        // authoritative source for the JS-redirect destination.
+        let page = null;
+        try {
+            update('Creating page%dots%');
+            page = await browser.newPage();
+            await page.setUserAgent(this.#options.userAgent);
+            // HTTP-auth handling — two cooperating pieces, BOTH required:
+            //
+            // 1. `page.authenticate({user, pass})` (always, even with empty
+            //    strings) registers a Fetch-domain auth handler with
+            //    Chromium. With empty credentials it ALSO drains Chromium's
+            //    native HTTP-auth dialog without sending anything
+            //    privileged — the dialog cannot be captured by
+            //    `page.on('dialog')` (HTTP-auth is not a JS dialog) and
+            //    would otherwise hang the navigation until puppeteer's
+            //    timeout fires. With non-empty credentials it provides the
+            //    scope's auth so the in-scope navigation succeeds.
+            //
+            // 2. Stripping URL-embedded credentials from the navigation
+            //    target. **This is the credential-leak guard.** When the
+            //    URL we hand puppeteer carries `user:pass@host`, Chromium
+            //    promotes those credentials into its HTTP-auth cache
+            //    keyed by (scheme, host, port, realm). Subsequent
+            //    sub-resource requests issued from the same page —
+            //    including cross-origin requests to a different hostname
+            //    sharing the same IP / port (e.g. an embedded
+            //    `<img src="http://127.0.0.1:8010/…">` loaded from a
+            //    `localhost:8010` page) — get the cached `Authorization`
+            //    header re-attached by the network stack. The
+            //    `Fetch.authRequired` event never fires for these
+            //    pre-emptive attachments, so neither `page.authenticate`
+            //    nor any custom Fetch listener can filter them. The only
+            //    way to keep the cred out of the cross-origin request is
+            //    to make sure it never enters the cache in the first
+            //    place — hence stripping the URL before navigation.
+            //
+            // Verified by `scope-auth-leak.e2e.ts`: removing either piece
+            // causes that test to fail (without auth → main 401 hangs;
+            // without strip → scope cred leaks to off-scope sub-resource).
+            await page.authenticate({
+                username: url.username ?? '',
+                password: url.password ?? '',
+            });
+            // Re-parse from `withoutHashAndAuth` rather than mutating the
+            // re-parsed `url.href` object: ExURL pre-computes `href`,
+            // `withoutHash` and other derived strings at parse time, and
+            // post-hoc field assignment (`navigateUrl.username = ''`)
+            // leaves those derived strings stale. Anything downstream that
+            // reads `navigateUrl.href` (e.g. a future beholder bump that
+            // switches `page.goto` from `withoutHashAndAuth` to `href`)
+            // would silently get back the credentialed string — defeating
+            // the leak guard. Building the navigation URL from a known
+            // credential-free string guarantees every field is consistent.
+            const navigateUrl = parseUrl(url.withoutHashAndAuth) ?? url;
+            const scraper = new Scraper();
+            scraper.on('changePhase', createChangePhaseHandler({
+                emit: (event) => void this.emit('changePhase', event),
+                update,
+                formatLog: formatPhaseLog,
+                buffer: this.#pendingPhaseErrors,
+                urlHref: url.href,
+            }));
+            const result = await scraper.scrapeStart(page, navigateUrl, {
+                isExternal,
+                captureImages: !isExternal && this.#options.captureImages,
+                excludeKeywords: this.#options.excludeKeywords,
+                disableQueries: this.#options.disableQueries,
+                metadataOnly,
+                retries: this.#options.retry,
+                headCheckResult,
+            });
+            update('Closing browser%dots%');
+            // JS-redirect rescue capture: when `scrapeStart` catches a
+            // `#fetchData` throw internally (e.g. `Page.goto returned null`
+            // because a client-side `window.location.replace()` /
+            // meta-refresh fired), it returns `{ type: 'error', ... }`
+            // instead of re-throwing — so the `catch` arm below never
+            // sees those cases. Read `page.url()` here while `page` is
+            // still alive (finally still hasn't called `handleBrowserClose`)
+            // and attach it to the result so `#scrapePage` can fold the
+            // source into a redirect edge. Without this capture, the
+            // rescue path is dead for the most common failure shape it
+            // was designed to handle.
+            //
+            // `page.url()` itself can throw when the browser context died
+            // mid-scrape (target crashed, session killed). On failure we
+            // fall through with `postNavigationUrl` unset so the existing
+            // HEAD-chain rescue / normal error path takes over.
+            if (result.type === 'error') {
+                try {
+                    const postNavigationUrl = page.url();
+                    return { ...result, postNavigationUrl };
+                }
+                catch (urlReadError) {
+                    crawlerLog('Reading page.url() for JS-redirect detection failed on %s: %O', url.href, urlReadError);
+                }
+            }
+            return result;
+        }
+        catch (error) {
+            // JS-redirect rescue: when `scrapeStart` throws because
+            // `page.goto()` returned `null` (the symptom of a client-side
+            // `window.location.replace()` / meta-refresh navigating away
+            // before the original response materialised), `page.url()` still
+            // reports the destination Chromium ended up on. Capturing it
+            // here lets `#scrapePage` fold the source into a redirect edge
+            // instead of recording a hard `status = -1` — `Page.goto returned
+            // null` classifies as `protocol`, which is neither permanent nor
+            // a puppeteer-fallback kind, so without this rescue the page
+            // loops through `--retry-failed` forever with the same failure.
+            //
+            // `page.url()` itself can throw when the browser context is
+            // already torn down (target closed, session killed). Treat any
+            // such failure as "no extra information" and fall back to the
+            // normal error path — the existing redirect-edge fallback that
+            // keys off `headCheckResult.redirectPaths` may still rescue the
+            // page when the HEAD pre-flight resolved a chain.
+            let postNavigationUrl;
+            if (page) {
+                try {
+                    postNavigationUrl = page.url();
+                }
+                catch (urlReadError) {
+                    crawlerLog('Reading page.url() for JS-redirect detection failed on %s: %O', url.href, urlReadError);
+                }
+            }
+            return {
+                type: 'error',
+                resources: [],
+                error: {
+                    name: error instanceof Error ? error.name : 'Error',
+                    message: error instanceof Error ? error.message : String(error),
+                    stack: error instanceof Error ? error.stack : undefined,
+                    shutdown: true,
+                },
+                ...(postNavigationUrl === undefined ? {} : { postNavigationUrl }),
+            };
+        }
+        finally {
+            // handleBrowserClose force-kills the underlying Chromium when a
+            // graceful close() hangs (e.g. the session died mid-scrape) and
+            // guarantees the finally never throws, so the try-block's return
+            // value or caught error is never masked.
+            await handleBrowserClose(browser, url.href, crawlerLog);
+        }
+    }
     /**
      * The default maximum number of concurrent scraping processes.
      *