@nitpicker/crawler 0.8.0 → 0.11.0

package/lib/crawler/crawler.js

@@ -9,19 +9,29 @@ import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-awa
 import c from 'ansi-colors';
 import pkg from '../../package.json' with { type: 'json' };
 import { crawlerLog } from '../debug.js';
+import { createChangePhaseHandler } from './create-change-phase-handler.js';
+import { derivePageSource } from './derive-page-source.js';
+import { deriveResourceSource } from './derive-resource-source.js';
 import { detectPaginationPattern } from './detect-pagination-pattern.js';
+import { drainPhaseErrors } from './drain-phase-errors.js';
 import { fetchDestination } from './fetch-destination.js';
 import { findScopeEntry } from './find-scope-entry.js';
 import { formatCrawlProgress } from './format-crawl-progress.js';
 import { generatePredictedUrls } from './generate-predicted-urls.js';
+import { handleBrowserClose } from './handle-browser-close.js';
 import { handleIgnoreAndSkip } from './handle-ignore-and-skip.js';
 import { handleResourceResponse } from './handle-resource-response.js';
 import { handleScrapeEnd } from './handle-scrape-end.js';
 import { handleScrapeError } from './handle-scrape-error.js';
 import { injectScopeAuth } from './inject-scope-auth.js';
+import { isHtmlContentType } from './is-html-content-type.js';
 import LinkList from './link-list.js';
 import { linkToPageData } from './link-to-page-data.js';
+import { logUndrainedPhaseErrors } from './log-undrained-phase-errors.js';
+import { partitionUrlsByHtml } from './partition-urls-by-html.js';
 import { protocolAgnosticKey } from './protocol-agnostic-key.js';
+import { redirectDestKey } from './redirect-dest-key.js';
+import { resourceToPageData } from './resource-to-page-data.js';
 import { RobotsChecker } from './robots-checker.js';
 import { shouldDiscardPredicted } from './should-discard-predicted.js';
 import { shouldSkipUrl } from './should-skip-url.js';
@@ -43,8 +53,18 @@ class Crawler extends EventEmitter {
     #linkList = new LinkList();
     /** Merged crawler configuration (user overrides + defaults). */
     #options;
+    /**
+     * Phase errors observed during {@link Crawler.#launchBrowserAndScrape},
+     * buffered per URL href so they can be emitted as `pageError` events
+     * AFTER the corresponding `page` / `externalPage` event. This ordering
+     * lets the orchestrator's WriteQueue serialise `setPage` before
+     * `insertPageError`, so the FK resolution via URL always finds the row.
+     */
+    #pendingPhaseErrors = new Map();
     /** Set of resource URLs (without hash) already captured, for deduplication. */
     #resources = new Set();
+    /** Number of HTML pages (isTarget=1) scraped in previous sessions, used to seed the progress counter on resume. */
+    #resumedPagesScraped = 0;
     /** URLs restored from a previous session that still need to be scraped. */
     #resumedPending = [];
     /** URLs already scraped in a previous session, used to populate the `seen` set in {@link #runDeal}. */
@@ -53,6 +73,13 @@ class Crawler extends EventEmitter {
     #robotsChecker;
     /** Maps hostnames to their scope URLs. Defines the crawl boundary for internal/external classification. */
     #scope = new Map();
+    /**
+     * Protocol-agnostic keys of redirect destinations already rendered (and stored)
+     * during this crawl. When many URLs redirect to one destination, only the first
+     * renders it; the rest record the redirect edge and skip the browser (#73).
+     * Keyed by {@link redirectDestKey}. Reset at the start of {@link #runDeal}.
+     */
+    #scrapedDestinations = new Set();
     /**
      * The AbortSignal associated with this crawler's AbortController.
      *
@@ -87,6 +114,8 @@ class Crawler extends EventEmitter {
             verbose: options?.verbose ?? false,
             userAgent: options?.userAgent || `Nitpicker/${pkg.version}`,
             ignoreRobots: options?.ignoreRobots ?? false,
+            lookupResource: options?.lookupResource ?? null,
+            inventoryMode: options?.inventoryMode ?? null,
         };
         this.#robotsChecker = new RobotsChecker(this.#options.userAgent, !this.#options.ignoreRobots);
         for (const urlStr of this.#options.roots) {
@@ -126,10 +155,14 @@ class Crawler extends EventEmitter {
      * @param pending - URLs that were pending (not yet scraped) in the previous session.
      * @param scraped - URLs that were already scraped in the previous session.
      * @param resources - Resource URLs that were already captured in the previous session.
+     * @param pagesScrapedOffset - Number of HTML pages already rendered in previous
+     *   sessions, used to seed the session-spanning progress counter. Defaults to 0
+     *   for callers that don't need cross-session accuracy in the progress display.
      */
-    resume(pending, scraped, resources) {
+    resume(pending, scraped, resources, pagesScrapedOffset = 0) {
         this.#resumedPending = this.#linkList.resume(pending, scraped, this.#options);
         this.#resumedScraped = scraped;
+        this.#resumedPagesScraped = pagesScrapedOffset;
         for (const resource of resources) {
             this.#resources.add(resource);
         }
@@ -170,7 +203,12 @@ class Crawler extends EventEmitter {
             this.#options.recursive = false;
             this.#options.fromList = true;
         }
-        const isResuming = this.#resumedScraped.length > 0;
+        // A resume can have an empty scraped set — e.g. a crawl interrupted before
+        // any page finished, or a `--retry-failed` run where every page in the
+        // archive was a failure and got reset to pending. Keying purely on
+        // `#resumedScraped` would then mistake the session for a fresh crawl and
+        // drop every resumed pending URL, so honour the pending set too.
+        const isResuming = this.#resumedScraped.length > 0 || this.#resumedPending.length > 0;
         // Dedupe by the same protocol-agnostic key the dealer uses internally.
         // Append-mode in particular can put the same URL into both
         // `#resumedPending` (via `repromoteExternalPages`) and `urls` (the
@@ -186,17 +224,38 @@ class Crawler extends EventEmitter {
             initialUrls.push(url);
         }
         const resumeOffset = this.#resumedScraped.length;
+        const pagesScrapedOffset = this.#resumedPagesScraped;
         if (initialUrls.length === 0) {
             crawlerLog('Crawl End (nothing to resume)');
             void this.emit('crawlEnd', {});
             return;
         }
-        void this.#runDeal(initialUrls, resumeOffset).catch((error) => {
+        void this.#runDeal(initialUrls, resumeOffset, pagesScrapedOffset).catch((error) => {
             crawlerLog('runDeal error: %O', error);
             this.#emitDealErrors(error, root.href);
             void this.emit('crawlEnd', {});
         });
     }
+    /**
+     * Thin instance-bound adapter over {@link drainPhaseErrors}. Flushes
+     * `#pendingPhaseErrors` for `url` as `pageError` events. Idempotent.
+     *
+     * **Test gap (known)**: this adapter is invoked from the worker body in
+     * {@link Crawler.#runDeal} at three call sites — after `#handleResult`,
+     * inside the worker's `catch`, and via `logUndrainedPhaseErrors` in
+     * `finally`. The drain logic itself is unit-tested in
+     * `drain-phase-errors.spec.ts`; the wiring (whether the worker actually
+     * calls it on each path) is verified by code review only, because
+     * driving the worker requires a Puppeteer + beholder mock stack whose
+     * cost outweighs the regression it would catch.
+     * @param url - URL whose buffered errors should be flushed.
+     * @param isExternal - Whether the URL is external to the crawl scope.
+     */
+    #drainPhaseErrors(url, isExternal) {
+        drainPhaseErrors(this.#pendingPhaseErrors, url.href, isExternal, (payload) => {
+            void this.emit('pageError', payload);
+        });
+    }
     /**
      * Emits error events for a deal-level failure.
      *
@@ -224,11 +283,17 @@ class Crawler extends EventEmitter {
      * @param resources - Sub-resource entries captured during the page load
      */
     #handleResources(resources) {
+        // `deriveResourceSource` encodes the "sub-resources are never seeds"
+        // rule and stays in lockstep with `derivePageSource` if PageSource
+        // gains new variants. Computed once outside the loop because the
+        // inventoryMode reference does not change mid-batch.
+        const subResourceSource = deriveResourceSource(this.#options.inventoryMode);
         for (const { resource, pageUrl } of resources) {
             const { isNew } = handleResourceResponse(resource, this.#resources);
             if (isNew) {
                 void this.emit('response', {
                     resource: resource,
+                    source: subResourceSource,
                 });
             }
             void this.emit('responseReferrers', {
@@ -246,20 +311,22 @@ class Crawler extends EventEmitter {
      * - `error`: Creates a fallback PageData, marks as done, and emits `error`.
      * @param result - The scrape result from beholder
      * @param url - The URL that was scraped
-     * @param push - Dealer's push callback to enqueue newly discovered URLs
+     * @param enqueue - Callback to enqueue newly discovered URLs into the dealer
+     *   queue, prioritising likely-HTML URLs to the front (see {@link partitionUrlsByHtml}).
+     *   Accepts a batch so a group of URLs (e.g. predicted pagination) keeps its order.
      * @param paginationState - Mutable state for predicted pagination cascade prevention
      * @param paginationState.lastPushedUrl
      * @param paginationState.lastPushedWasPredicted
      * @param concurrency - Current concurrency level, used to determine predicted URL count
      */
-    #handleResult(result, url, push, paginationState, concurrency) {
+    #handleResult(result, url, enqueue, paginationState, concurrency) {
         switch (result.type) {
             case 'success': {
                 if (!result.pageData)
                     break;
                 handleScrapeEnd(result.pageData, this.#linkList, this.#scope, this.#options, (newUrl, opts) => {
                     this.#linkList.add(newUrl, opts);
-                    void push(newUrl);
+                    void enqueue(newUrl);
                     // Predicted pagination detection
                     if (!paginationState || !concurrency)
                         return;
@@ -276,13 +343,18 @@ class Crawler extends EventEmitter {
                         const pattern = detectPaginationPattern(paginationState.lastPushedUrl, newUrl.withoutHashAndAuth);
                         if (pattern) {
                             const urls = generatePredictedUrls(pattern, newUrl.withoutHashAndAuth, concurrency);
+                            const specUrls = [];
                             for (const specUrlStr of urls) {
                                 const specUrl = parseUrl(specUrlStr, this.#options);
                                 if (specUrl) {
                                     this.#linkList.add(specUrl, { predicted: true });
-                                    void push(specUrl);
+                                    specUrls.push(specUrl);
                                 }
                             }
+                            // Enqueue as one batch so ascending page order is kept
+                            // at the front of the queue (see enqueue in #runDeal).
+                            if (specUrls.length > 0)
+                                void enqueue(...specUrls);
                             paginationState.lastPushedUrl = newUrl.withoutHashAndAuth;
                             paginationState.lastPushedWasPredicted = true;
                             return;
@@ -291,11 +363,20 @@ class Crawler extends EventEmitter {
                     paginationState.lastPushedUrl = newUrl.withoutHashAndAuth;
                     paginationState.lastPushedWasPredicted = false;
                 });
-                if (result.pageData.isExternal) {
-                    void this.emit('externalPage', { result: result.pageData });
-                }
-                else {
-                    void this.emit('page', { result: result.pageData });
+                {
+                    const pageSource = derivePageSource(this.#options.inventoryMode, result.pageData.url.withoutHashAndAuth);
+                    if (result.pageData.isExternal) {
+                        void this.emit('externalPage', {
+                            result: result.pageData,
+                            source: pageSource,
+                        });
+                    }
+                    else {
+                        void this.emit('page', {
+                            result: result.pageData,
+                            source: pageSource,
+                        });
+                    }
                 }
                 break;
             }
@@ -324,11 +405,15 @@ class Crawler extends EventEmitter {
                 }, this.#linkList, this.#scope, this.#options);
                 const isExternal = findScopeEntry(url, this.#scope, this.#options) === null;
                 if (pageResult) {
+                    const pageSource = derivePageSource(this.#options.inventoryMode, pageResult.url.withoutHashAndAuth);
                     if (pageResult.isExternal) {
-                        void this.emit('externalPage', { result: pageResult });
+                        void this.emit('externalPage', {
+                            result: pageResult,
+                            source: pageSource,
+                        });
                     }
                     else {
-                        void this.emit('page', { result: pageResult });
+                        void this.emit('page', { result: pageResult, source: pageSource });
                     }
                 }
                 void this.emit('error', {
@@ -384,13 +469,13 @@ class Crawler extends EventEmitter {
                 });
             }
             const scraper = new Scraper();
-            scraper.on('changePhase', (e) => {
-                const msg = formatPhaseLog(e);
-                if (msg) {
-                    update(msg);
-                }
-                void this.emit('changePhase', e);
-            });
+            scraper.on('changePhase', createChangePhaseHandler({
+                emit: (event) => void this.emit('changePhase', event),
+                update,
+                formatLog: formatPhaseLog,
+                buffer: this.#pendingPhaseErrors,
+                urlHref: url.href,
+            }));
             const result = await scraper.scrapeStart(page, url, {
                 isExternal,
                 captureImages: !isExternal && this.#options.captureImages,
@@ -416,7 +501,11 @@ class Crawler extends EventEmitter {
             };
         }
         finally {
-            await browser.close().catch(() => { });
+            // handleBrowserClose force-kills the underlying Chromium when a
+            // graceful close() hangs (e.g. the session died mid-scrape) and
+            // guarantees the finally never throws, so the try-block's return
+            // value or caught error is never masked.
+            await handleBrowserClose(browser, url.href, crawlerLog);
         }
     }
     /**
@@ -429,16 +518,25 @@ class Crawler extends EventEmitter {
      * @param initialUrls - Starting URLs to seed the deal queue
      * @param resumeOffset - Number of URLs already scraped in a previous session,
      *   added to the progress counter for accurate display
+     * @param pagesScrapedOffset - Number of HTML pages already rendered in previous
+     *   sessions, used to seed the per-session HTML-pages counter so the display
+     *   remains accurate across resumes
      */
-    async #runDeal(initialUrls, resumeOffset = 0) {
+    async #runDeal(initialUrls, resumeOffset = 0, pagesScrapedOffset = 0) {
         const seen = new Set(initialUrls.map((u) => protocolAgnosticKey(u.withoutHashAndAuth)));
         // Add scraped URLs to seen to prevent re-processing during resume
         for (const url of this.#resumedScraped) {
             seen.add(protocolAgnosticKey(url));
         }
+        // Redirect-destination dedup is per-crawl; clear any state from a prior run.
+        this.#scrapedDestinations.clear();
         // external URL の追跡（target は deal の total/done から導出）
         const externalUrls = new Set();
         const externalDoneUrls = new Set();
+        // HTML ページとしてブラウザでレンダリングし、かつアーカイブに保存されたページ数。
+        // HEAD のみ・title 取得のみ・skip・ブラウザ起動失敗・predicted-discard は含まない。
+        // 過去セッションぶんは pagesScrapedOffset として init される。
+        let pagesScraped = pagesScrapedOffset;
         // 初期 URL を分類（onPush を通らないため）
         for (const url of initialUrls) {
             if (findScopeEntry(url, this.#scope, this.#options) === null) {
@@ -453,7 +551,7 @@ class Crawler extends EventEmitter {
             lastPushedUrl: null,
             lastPushedWasPredicted: false,
         };
-        await deal(initialUrls, (url, update, _index, setLineHeader, push) => {
+        await deal(initialUrls, (url, update, _index, setLineHeader, push, unshift) => {
             const matchedScope = findScopeEntry(url, this.#scope, this.#options);
             const isExternal = matchedScope === null;
             const urlText = isExternal ? c.dim(url.href) : c.cyan(url.href);
@@ -463,8 +561,30 @@ class Crawler extends EventEmitter {
             }
             this.#linkList.add(url);
             this.#linkList.progress(url);
+            // Likely-HTML URLs jump to the front of the queue (unshift) so page
+            // crawling advances ahead of asset/document fetches; everything else
+            // is appended (push). partitionUrlsByHtml splits the batch by the
+            // URL-only heuristic. Variadic so a batch (e.g. predicted pagination)
+            // keeps its order: a single unshift(...html) preserves ascending order
+            // at the front, whereas unshifting one-by-one would reverse it.
+            const enqueue = (...newUrls) => {
+                const [html, other] = partitionUrlsByHtml(newUrls);
+                const ops = [];
+                if (html.length > 0)
+                    ops.push(unshift(...html));
+                if (other.length > 0)
+                    ops.push(push(...other));
+                return Promise.all(ops).then(() => { });
+            };
             return async () => {
                 const log = createTimedUpdate(update, this.#options.verbose);
+                // `#scrapePage` 内のブラウザ HTML レンダーが成功したかをマークするフラグ。
+                // 成功時のみ #scrapePage 側で true に設定される。
+                // discard 判定後にこのフラグを見てカウントするので、launch 失敗や predicted-discard は除外される。
+                let renderedInBrowser = false;
+                const markBrowserScrape = () => {
+                    renderedInBrowser = true;
+                };
                 try {
                     const robotsAllowed = await this.#robotsChecker.isAllowed(url);
                     if (!robotsAllowed) {
@@ -496,24 +616,56 @@ class Crawler extends EventEmitter {
                             isLowerLayer: false,
                         });
                         this.#linkList.done(url, this.#scope, { page: pageData }, this.#options);
-                        void this.emit('externalPage', { result: pageData });
+                        void this.emit('externalPage', {
+                            result: pageData,
+                            source: derivePageSource(this.#options.inventoryMode, url.withoutHashAndAuth),
+                        });
                         log(c.dim('External (skip fetch)'));
                         return;
                     }
                     const metadataOnly = this.#linkList.isMetadataOnly(url.withoutHash);
                     const isPredicted = this.#linkList.isPredicted(url.withoutHashAndAuth);
                     log('Scraping%dots%');
-                    const result = await this.#scrapePage(url, log, metadataOnly, _index);
+                    const result = await this.#scrapePage(url, log, metadataOnly, _index, markBrowserScrape);
+                    // Redirect convergence (#73): the destination was already
+                    // rendered during this crawl, so only the redirect edge is
+                    // recorded and the browser was never launched. Mark the URL
+                    // done and emit `redirect` (routed to `Archive.setRedirect`,
+                    // which writes the edge without touching the destination's
+                    // content). This URL does not count toward pagesScraped.
+                    if (result.type === 'redirect-edge') {
+                        // Note: a predicted (speculative) URL that reaches here genuinely
+                        // redirects (the server returned 3xx), so it is a real URL — we
+                        // record its edge rather than discard it. This matches the render
+                        // path, where the first predicted source to a destination renders
+                        // it and is recorded as a redirect source the same way; only 404 /
+                        // error predicted URLs are dropped (by `shouldDiscardPredicted`).
+                        this.#linkList.done(url, this.#scope, { page: result.pageData }, this.#options);
+                        void this.emit('redirect', { result: result.pageData });
+                        log(c.dim('Redirect (dest already scraped)'));
+                        return;
+                    }
                     // Discard predicted URLs that failed (404, error, etc.)
                     if (isPredicted && shouldDiscardPredicted(result)) {
                         handleIgnoreAndSkip(url, this.#linkList, this.#scope, this.#options);
                         log(c.dim('Predicted (discarded)'));
                         return;
                     }
+                    // Count only after discard check: rendered HTML pages that
+                    // will be persisted to the archive. Launch failures bypass
+                    // this point via the catch block; discarded predicted URLs
+                    // return above without reaching here.
+                    if (renderedInBrowser) {
+                        pagesScraped++;
+                    }
                     log('Saving results%dots%');
-                    this.#handleResult(result, url, push, paginationState, concurrency);
+                    this.#handleResult(result, url, enqueue, paginationState, concurrency);
                     this.#handleResources(result.resources);
                     log(formatResultSummary(result));
+                    // Phase errors must be emitted AFTER 'page' / 'externalPage'
+                    // so the orchestrator's WriteQueue sees `setPage` before
+                    // `insertPageError` and the URL→pageId resolution succeeds.
+                    this.#drainPhaseErrors(url, isExternal);
                 }
                 catch (error) {
                     crawlerLog('Worker error for %s: %O', url.href, error);
@@ -532,11 +684,21 @@ class Crawler extends EventEmitter {
                         isExternal,
                         error: workerError,
                     });
+                    // Hard-error path: persist whatever phase errors we have
+                    // already buffered so they are not lost.
+                    this.#drainPhaseErrors(url, isExternal);
                 }
                 finally {
                     if (isExternal) {
                         externalDoneUrls.add(protocolAgnosticKey(url.withoutHashAndAuth));
                     }
+                    // Phase errors still in the buffer here were not drained
+                    // by the success or catch paths — typically because a
+                    // predicted URL was discarded before reaching the drain
+                    // point. The helper logs the drop (observable via
+                    // DEBUG=Nitpicker:Crawler) and removes the entry so the
+                    // Map cannot leak across crawls.
+                    logUndrainedPhaseErrors(this.#pendingPhaseErrors, url.href, crawlerLog);
                 }
             };
         }, {
@@ -551,6 +713,7 @@ class Crawler extends EventEmitter {
                     resumeOffset,
                     externalTotal: externalUrls.size,
                     externalDone: externalDoneUrls.size,
+                    pagesScraped,
                     limit,
                 });
             },
@@ -581,14 +744,56 @@ class Crawler extends EventEmitter {
      * @param update - Callback for progress messages
      * @param metadataOnly - When true, only extract title metadata without full browser scraping
      * @param laneIndex - The dealer lane index, used to create unique countdown IDs
+     * @param markBrowserScrape - Called once **after** the browser successfully
+     *   renders an HTML page (i.e. `#launchBrowserAndScrape` resolved with
+     *   `type: 'success'`). Not called for HEAD-only, title-only, captured-resource
+     *   reuse, non-HTML responses, non-HTTP protocols (mailto:, tel:), browser
+     *   launch throws (e.g. invalid executablePath), or scraper-returned
+     *   `type: 'error'` results. The caller is responsible for further filtering
+     *   (e.g. predicted-discard).
      * @returns The scrape result
      */
-    async #scrapePage(url, update, metadataOnly, laneIndex) {
+    async #scrapePage(url, update, metadataOnly, laneIndex, markBrowserScrape) {
         const isExternal = findScopeEntry(url, this.#scope, this.#options) === null;
         // Non-HTTP protocols (mailto:, tel:, etc.) — let the scraper handle early return
         if (!url.isHTTP) {
             return this.#launchBrowserAndScrape(url, update, isExternal, metadataOnly);
         }
+        // Reuse captured resource data — when this URL was already observed as a
+        // sub-resource during page rendering, its response data is recorded and
+        // the HEAD pre-flight is redundant. Only 2xx non-HTML rows are eligible
+        // (see resourceToPageData); anything else falls through to the pre-flight.
+        // Both URL variants are checked because scope-auth injection adds
+        // credentials to queued URLs while browser-captured resource URLs have none.
+        // The result is deliberately NOT written to destinationCache: a queued URL
+        // is processed at most once (the dealer dedupes by protocol-agnostic key),
+        // so a URL that takes this path never reaches fetchDestination again.
+        const lookupResource = this.#options.lookupResource;
+        if (lookupResource &&
+            (this.#resources.has(url.withoutHash) ||
+                this.#resources.has(url.withoutHashAndAuth))) {
+            update('Checking captured resource%dots%');
+            let resource = null;
+            try {
+                resource = await lookupResource([url.withoutHash, url.withoutHashAndAuth]);
+            }
+            catch (error) {
+                // A lookup failure must never be worse than not having the
+                // optimization — fall back to the HEAD pre-flight below.
+                crawlerLog('Resource lookup failed for %s, falling back: %O', url.href, error);
+            }
+            const pageData = resource
+                ? resourceToPageData({ url, isExternal, resource })
+                : null;
+            if (pageData) {
+                crawlerLog('Reused captured resource for %s', url.href);
+                return {
+                    type: 'success',
+                    pageData: metadataOnly ? { ...pageData, isTarget: false } : pageData,
+                    resources: [],
+                };
+            }
+        }
         // Pre-flight: lightweight HEAD request to check server availability
         update('HEAD request%dots%');
         let headCheckResult;
@@ -609,10 +814,32 @@ class Crawler extends EventEmitter {
                 },
             };
         }
+        // Redirect convergence (#73): `finalKey` is the destination this URL lands
+        // on after following its redirect chain (or the URL itself when it does not
+        // redirect). When that destination has already been rendered and stored
+        // during this crawl, do NOT process this URL further — record the redirect
+        // edge only and skip everything below, regardless of content type. This is
+        // the root fix for the many-to-one redirect duplication (#70): every source
+        // URL that 301s to one destination otherwise re-renders/re-stores it. The
+        // check sits ABOVE the metadata-only and non-HTML branches on purpose — both
+        // route their HEAD/title result through `updatePage`, which would funnel a
+        // content-less result into `#insertPage` and overwrite the already-rendered
+        // destination's title / meta / isExternal. The edge-only path leaves the
+        // destination row intact.
+        //
+        // `finalKey` is also claimed for destinations reached directly (no redirect;
+        // see the claim after a successful render below), so a destination that is
+        // both linked directly and arrived at via a redirect is rendered by whichever
+        // path wins the race, not both.
+        const finalKey = redirectDestKey(url, headCheckResult.redirectPaths);
+        if (this.#scrapedDestinations.has(finalKey)) {
+            crawlerLog('Redirect dest already rendered, edge only: %s', url.href);
+            return { type: 'redirect-edge', pageData: headCheckResult };
+        }
         // Title-only mode — extract <title> via partial GET for HTML, skip browser
         if (metadataOnly) {
             if (headCheckResult.contentType === null ||
-                headCheckResult.contentType === 'text/html') {
+                isHtmlContentType(headCheckResult.contentType)) {
                 update('Fetching title%dots%');
                 try {
                     const titleResult = await fetchDestination({
@@ -640,15 +867,46 @@ class Crawler extends EventEmitter {
         }
         // Non-HTML content — skip browser
         if (headCheckResult.contentType !== null &&
-            headCheckResult.contentType !== 'text/html') {
+            !isHtmlContentType(headCheckResult.contentType)) {
             return {
                 type: 'success',
                 pageData: headCheckResult,
                 resources: [],
             };
         }
-        // HTML or unknown content type — launch browser with preflight result
-        return this.#launchBrowserAndScrape(url, update, isExternal, metadataOnly, headCheckResult);
+        // HTML or unknown content type — launch browser with preflight result.
+        // markBrowserScrape() fires only when the result is `success`.
+        // `#launchBrowserAndScrape` catches internal errors and returns
+        // `{ type: 'error', ... }` instead of throwing (see its catch block),
+        // so awaiting alone does NOT prove the page was rendered. The explicit
+        // success check excludes navigation failures, scraper exceptions, and
+        // shutdown-class errors from the pages-rendered count.
+        const browserResult = await this.#launchBrowserAndScrape(url, update, isExternal, metadataOnly, headCheckResult);
+        if (browserResult.type === 'success') {
+            markBrowserScrape();
+            // Claim the destination that was ACTUALLY rendered, keyed off the
+            // browser's own redirect resolution rather than the HEAD pre-flight's
+            // guess (`finalKey`). The browser is authoritative for what got stored;
+            // if HEAD and the browser disagree on the final URL (method-conditional
+            // / JS / meta-refresh redirects), keying the claim off the HEAD guess
+            // would route a sibling source to an edge pointing at a never-rendered
+            // phantom row. By claiming the rendered URL, a divergent sibling simply
+            // re-renders (dedup misses) instead — correct, just less optimal. In the
+            // common case HEAD and the browser agree, so the keys are identical.
+            //
+            // Claimed only after a successful render, so a failed render leaves the
+            // destination unclaimed and a later source retries it. Concurrent
+            // in-flight sources to the same destination (bounded by the concurrency
+            // limit) may still each render before any claim lands; the storage-layer
+            // replace in `updatePage` (#70) keeps the resulting anchors / images
+            // correct (sub-resources may briefly duplicate, far below the pre-#73
+            // once-per-source blow-up).
+            const renderedKey = browserResult.pageData
+                ? redirectDestKey(url, browserResult.pageData.redirectPaths)
+                : finalKey;
+            this.#scrapedDestinations.add(renderedKey);
+        }
+        return browserResult;
     }
     /**
      * Performs a pre-flight HTTP HEAD request with retry logic.

package/lib/crawler/create-change-phase-handler.d.ts

@@ -0,0 +1,54 @@
+import type { BufferedPhaseError } from './drain-phase-errors.js';
+import type { ChangePhaseEvent } from '@d-zero/beholder';
+/**
+ * Options for {@link createChangePhaseHandler}. Declared structurally so
+ * tests can pass plain `vi.fn()` stubs without spinning up a real Crawler.
+ */
+export interface ChangePhaseHandlerOptions {
+    /**
+     * Forwards the raw {@link ChangePhaseEvent} so external listeners on the
+     * Crawler still see every transition (typically `this.emit.bind(this)`
+     * narrowed to the `changePhase` channel).
+     */
+    emit: (event: ChangePhaseEvent) => void;
+    /** Receives the formatted progress log line. Skipped when empty. */
+    update: (log: string) => void;
+    /**
+     * Renders the human-readable progress message for an event. Injected so
+     * the handler stays free of the Crawler's internal log formatter.
+     * Returns `null` for events that should not surface to `update`.
+     */
+    formatLog: (event: ChangePhaseEvent) => string | null;
+    /**
+     * Per-URL buffer of `retryExhausted` failures. The handler appends to
+     * this map; it does not drain (that is `drainPhaseErrors`'s job).
+     */
+    buffer: Map<string, BufferedPhaseError[]>;
+    /** URL href used as the buffer key for this scrape. */
+    urlHref: string;
+}
+/**
+ * Builds the `scraper.on('changePhase', ...)` listener used by
+ * {@link Crawler.#launchBrowserAndScrape}.
+ *
+ * Three responsibilities:
+ * 1. Render the phase log via the injected `formatLog` and pipe it to `update`.
+ * 2. Forward the raw event so external consumers (CLI progress UI etc.) see
+ *    every transition.
+ * 3. Buffer `retryExhausted` events into the per-URL phase-error map so they
+ *    can be drained as `pageError` events AFTER the `page` event fires.
+ *
+ * WHY a factory: the listener captures per-scrape state (`buffer`, `urlHref`,
+ * `update`). Extracting the factory makes the wiring directly unit-testable
+ * with plain stubs, instead of requiring a mocked Puppeteer + beholder + dealer
+ * stack to drive the worker.
+ *
+ * **Caller contract**: register the returned handler at most once per
+ * `scraper` instance. The Crawler creates a fresh Scraper per URL so this
+ * holds today; if scraper pooling is ever introduced, register exactly one
+ * handler per scrape and unregister it on completion to avoid duplicate
+ * buffer entries.
+ * @param options - Wiring dependencies for the handler.
+ * @returns A function suitable for `scraper.on('changePhase', ...)`.
+ */
+export declare function createChangePhaseHandler(options: ChangePhaseHandlerOptions): (event: ChangePhaseEvent) => void;