@nitpicker/crawler 0.15.0 → 0.17.0

package/lib/archive/types.d.ts

@@ -121,6 +121,7 @@ export type PageSource = 'crawled' | 'inventory-seed' | 'inventory-discovered';
  *   new_pages: 1234,
  *   new_resources: 56,
  *   scope_skipped: 7,
+ *   exclude_skipped: 3,
  * });
  */
 export interface InventoryRunMeta {
@@ -138,6 +139,8 @@ export interface InventoryRunMeta {
     new_resources?: number | null;
     /** Number of input URLs dropped because they fell outside the archived scope. */
     scope_skipped?: number | null;
+    /** Number of novel in-scope input URLs recorded as terminal skipped pages (`is_skipped=1`, `skip_reason='excluded'`) instead of being imported, because they matched the effective `excludes` / `excludeUrls` config. Pure audit output like every other count on this row — written once per run, read back only by `listInventoryRuns` display surfaces, never consumed by any runtime decision. `null` on rows written before the column existed (those runs predate ingestion-side exclusion — their excluded URLs were imported as real pages/resources, not counted). */
+    exclude_skipped?: number | null;
     /** Number of source-file lines dropped by the CLI for failing URL validation, before this row's `total_lines` was counted. `null` for programmatic callers that built the URL list in-memory (no source file to have invalid lines). */
     invalid_skipped?: number | null;
     /** Free-form text for backfill annotations or operator notes. */
@@ -186,6 +189,19 @@ export interface InsertNetworkOutageParams {
     triggerErrorCount: number;
     triggerHostCount: number;
 }
+/**
+ * Fields required to record a newly-capped URL shape via
+ * `Database.insertDedupeCapEvent`. camelCase, mapped to snake_case columns
+ * on write — same convention as {@link InsertNetworkOutageParams}.
+ */
+export interface InsertDedupeCapEventParams {
+    shapeKey: string;
+    sampleUrl: string;
+    bodyHash: Buffer;
+    effectiveThreshold: number;
+    observedCount: number;
+    detectedAt: number;
+}
 /**
  * Filter type for querying pages from the database.
  *

package/lib/classify-error-kind.d.ts

@@ -14,6 +14,7 @@ import type { ErrorKind } from './types.js';
  * classifyErrorKind('getaddrinfo ENOTFOUND www.example.com'); // 'dns'
  * classifyErrorKind('gave up after 3 retries — Race 180,000ms'); // 'timeout'
  * classifyErrorKind('Protocol error (Page.reload): Target closed'); // 'protocol'
+ * classifyErrorKind('Maximum number of redirects exceeded'); // 'redirect-loop'
  * ```
  */
 export declare function classifyErrorKind(message: string): ErrorKind;

package/lib/classify-error-kind.js

@@ -53,6 +53,19 @@ const MATCHERS = [
         kind: 'parse-error',
         pattern: /Parse Error|Expected HTTP\/|Unexpected end of stream/i,
     },
+    // `follow-redirects` (pinned at 1.16.0, `fetch-destination.ts`'s HEAD/GET
+    // pre-flight) throws exactly "Maximum number of redirects exceeded" —
+    // no cause token, no URL — when a chain never terminates within its
+    // `maxRedirects` budget. `ERR_TOO_MANY_REDIRECTS` is the equivalent
+    // Chromium net-error code, covering the same symptom surfaced through a
+    // puppeteer navigation instead of the Node HTTP client. Both mean the
+    // SAME thing: the site's own redirect configuration loops and will loop
+    // again on any future fetch, which is why this is deterministic (not a
+    // transient network condition) — see `PERMANENT_ERROR_KINDS`.
+    {
+        kind: 'redirect-loop',
+        pattern: /Maximum number of redirects exceeded|ERR_TOO_MANY_REDIRECTS/i,
+    },
     // `client-blocked` covers Chromium's ERR_BLOCKED_* family — the browser
     // actively decided to reject the request (ad/tracker heuristics, CSP,
     // CORB / ORB, administrator block list, fingerprinting protection,
@@ -110,6 +123,7 @@ const MATCHERS = [
  * classifyErrorKind('getaddrinfo ENOTFOUND www.example.com'); // 'dns'
  * classifyErrorKind('gave up after 3 retries — Race 180,000ms'); // 'timeout'
  * classifyErrorKind('Protocol error (Page.reload): Target closed'); // 'protocol'
+ * classifyErrorKind('Maximum number of redirects exceeded'); // 'redirect-loop'
  * ```
  */
 export function classifyErrorKind(message) {

package/lib/crawler/assert-chrome-installed.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Verifies that Puppeteer can resolve an installed Chrome/Chromium executable
+ * before a crawl starts.
+ *
+ * A crawl otherwise only discovers a missing browser deep inside the
+ * per-URL scrape loop (`Crawler#_launchBrowserAndScrape`), where it surfaces
+ * as one more scrape error among many — the CLI still prints "Crawl
+ * completed" and writes an archive, so a missing Chrome (a fatal
+ * precondition, not a per-page failure) is easy to miss. Calling this once,
+ * before any archive I/O begins, turns it into an immediate, actionable
+ * failure instead.
+ * @param executablePath - Explicit override, matching
+ *   {@link CrawlerOptions.executablePath}. Pass `null` (or omit) to check
+ *   Puppeteer's own pinned Chrome resolution instead.
+ * @throws {Error} When the resolved executable path does not exist on disk.
+ * @example
+ * ```ts
+ * import { assertChromeIsInstalled } from '@nitpicker/crawler';
+ *
+ * // Throws with install instructions before any crawl work starts.
+ * await assertChromeIsInstalled();
+ * ```
+ */
+export declare function assertChromeIsInstalled(executablePath?: string | null): Promise<void>;

package/lib/crawler/assert-chrome-installed.js

@@ -0,0 +1,43 @@
+import { existsSync } from 'node:fs';
+import path from 'node:path';
+import pkg from '../../package.json' with { type: 'json' };
+/**
+ * Verifies that Puppeteer can resolve an installed Chrome/Chromium executable
+ * before a crawl starts.
+ *
+ * A crawl otherwise only discovers a missing browser deep inside the
+ * per-URL scrape loop (`Crawler#_launchBrowserAndScrape`), where it surfaces
+ * as one more scrape error among many — the CLI still prints "Crawl
+ * completed" and writes an archive, so a missing Chrome (a fatal
+ * precondition, not a per-page failure) is easy to miss. Calling this once,
+ * before any archive I/O begins, turns it into an immediate, actionable
+ * failure instead.
+ * @param executablePath - Explicit override, matching
+ *   {@link CrawlerOptions.executablePath}. Pass `null` (or omit) to check
+ *   Puppeteer's own pinned Chrome resolution instead.
+ * @throws {Error} When the resolved executable path does not exist on disk.
+ * @example
+ * ```ts
+ * import { assertChromeIsInstalled } from '@nitpicker/crawler';
+ *
+ * // Throws with install instructions before any crawl work starts.
+ * await assertChromeIsInstalled();
+ * ```
+ */
+export async function assertChromeIsInstalled(executablePath) {
+    if (executablePath) {
+        const execPath = path.resolve(executablePath);
+        if (existsSync(execPath)) {
+            return;
+        }
+        throw new Error(`Executable path does not exist: ${execPath}`);
+    }
+    const puppeteer = await import('puppeteer');
+    const resolvedPath = await puppeteer.executablePath();
+    if (existsSync(resolvedPath)) {
+        return;
+    }
+    const puppeteerVersion = pkg.dependencies.puppeteer;
+    throw new Error(`Chrome executable not found at: ${resolvedPath}\n` +
+        `Run \`npx puppeteer@${puppeteerVersion} browsers install chrome\` to install the Chrome build Puppeteer expects, then retry.`);
+}

package/lib/crawler/crawler.d.ts

@@ -37,6 +37,18 @@ export default class Crawler extends EventEmitter<CrawlerEventTypes> {
      * {@link #runDeal}.
      */
     abort(): void;
+    /**
+     * Per-shape count of anchors the dedupe-cap enqueue gates rejected after
+     * that shape capped (opt-in `--dedupe-cap`). Read by
+     * `CrawlerOrchestrator` at `crawlEnd` to finalize each
+     * `dedupe_cap_events.rejected_count` exactly once — rejections are
+     * accumulated in memory rather than written to the archive per-rejection
+     * to avoid write amplification (a capped trap can generate an unbounded
+     * number of rejected anchors).
+     * @returns A snapshot of the per-shape rejection counts. Empty when
+     *   `--dedupe-cap` was not enabled or no shape has capped yet.
+     */
+    getDedupeCapRejections(): ReadonlyMap<string, number>;
     /**
      * Retrieve the list of Chromium process IDs that are still running.
      *

package/lib/crawler/crawler.js

@@ -9,6 +9,7 @@ import { retryCall } from '@d-zero/shared/retry';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
 import c from 'ansi-colors';
 import pkg from '../../package.json' with { type: 'json' };
+import { computeBodyHash } from '../archive/body-hash/compute-body-hash.js';
 import { classifyErrorKind } from '../classify-error-kind.js';
 import { crawlerLog } from '../debug.js';
 import { buildJsRedirectEdge } from './build-js-redirect-edge.js';
@@ -16,6 +17,12 @@ import { buildRedirectEvent } from './build-redirect-event.js';
 import { captureImageDomPaths } from './capture-image-dom-paths.js';
 import { chooseProbeHost } from './choose-probe-host.js';
 import { createChangePhaseHandler } from './create-change-phase-handler.js';
+import { decodeAuthCredential } from './decode-auth-credential.js';
+import { computeMetaSignature } from './dedupe/compute-meta-signature.js';
+import { computeShapeKey } from './dedupe/compute-shape-key.js';
+import DedupeCapTracker from './dedupe/dedupe-cap-tracker.js';
+import { isPredictedContentDuplicate } from './dedupe/is-predicted-content-duplicate.js';
+import { resolveOgUrlMismatch } from './dedupe/resolve-og-url-mismatch.js';
 import { derivePageSource } from './derive-page-source.js';
 import { destinationCache } from './destination-cache.js';
 import { detectPaginationPattern } from './detect-pagination-pattern.js';
@@ -72,6 +79,8 @@ const DEFAULT_NETWORK_OUTAGE_ERROR_THRESHOLD = 5;
 const DEFAULT_NETWORK_OUTAGE_HOST_THRESHOLD = 2;
 /** Default {@link CrawlerOptions.networkOutageProbeIntervalMs}. */
 const DEFAULT_NETWORK_OUTAGE_PROBE_INTERVAL_MS = 10_000;
+/** Default {@link CrawlerOptions.dedupeMapCap}. */
+const DEFAULT_DEDUPE_MAP_CAP = 100_000;
 /**
  * The core crawler engine that discovers and scrapes web pages.
  *
@@ -86,6 +95,24 @@ const DEFAULT_NETWORK_OUTAGE_PROBE_INTERVAL_MS = 10_000;
 class Crawler extends EventEmitter {
     /** Controller used to cancel the deal-based crawl via its AbortSignal. */
     #abortController = new AbortController();
+    /**
+     * Per-shape count of anchors rejected by the dedupe-cap enqueue gates
+     * after that shape capped. Read by {@link getDedupeCapRejections} at
+     * `crawlEnd` so the orchestrator can finalize each
+     * `dedupe_cap_events.rejected_count` exactly once (see
+     * `Crawler#getDedupeCapRejections`'s JSDoc for why this is not written
+     * to the archive incrementally).
+     */
+    #dedupeCapRejectionCounts = new Map();
+    /**
+     * Opt-in (`--dedupe-cap`) same-cluster soft cap. Always constructed
+     * (Misra-Gries state stays empty when {@link CrawlerOptions.dedupeCap} is
+     * `null`), gated on by `#options.dedupeCap !== null` at each call site
+     * rather than being conditionally `undefined`, so the two enqueue gates
+     * and the observation call in {@link #handleResult} do not need to
+     * null-check a class field.
+     */
+    #dedupeCapTracker;
     /** Tracks discovered URLs, their scrape status, and deduplication. */
     #linkList = new LinkList();
     /**
@@ -129,6 +156,23 @@ class Crawler extends EventEmitter {
      * `insertPageError`, so the FK resolution via URL always finds the row.
      */
     #pendingPhaseErrors = new Map();
+    /**
+     * Predicted-pagination body-hash tracking (always-on — independent of
+     * the opt-in `--dedupe-cap` tracker). Maps a URL shape key
+     * ({@link computeShapeKey}) to the {@link computeBodyHash} of the most
+     * recently scraped *predicted* page of that shape. Never reset mid-crawl
+     * (persists for the whole session, like {@link #scrapedDestinations}).
+     */
+    #predictedShapeBodyHashes = new Map();
+    /**
+     * Shapes for which {@link #predictedShapeBodyHashes} detected a
+     * content-duplicate predicted page (see {@link isPredictedContentDuplicate}).
+     * Once a shape lands here, no further predicted URLs are generated for it
+     * (checked in {@link #handleResult}'s pagination-pattern branch) — the
+     * cheapest possible way to stop a self-generating trap without needing
+     * the opt-in dedupe-cap machinery.
+     */
+    #predictedShapeStopped = new Set();
     /** Set of resource URLs (without hash) already captured, for deduplication. */
     #resources = new Set();
     /** Number of HTML pages (isTarget=1) scraped in previous sessions, used to seed the progress counter on resume. */
@@ -205,12 +249,16 @@ class Crawler extends EventEmitter {
             networkOutageHostThreshold: options?.networkOutageHostThreshold ?? DEFAULT_NETWORK_OUTAGE_HOST_THRESHOLD,
             networkOutageProbeIntervalMs: options?.networkOutageProbeIntervalMs ?? DEFAULT_NETWORK_OUTAGE_PROBE_INTERVAL_MS,
             networkProbe: options?.networkProbe ?? null,
+            dedupeCap: options?.dedupeCap ?? null,
+            dedupeMapCap: options?.dedupeMapCap ?? DEFAULT_DEDUPE_MAP_CAP,
+            preloadedStickyShapeKeys: options?.preloadedStickyShapeKeys ?? [],
         };
         this.#networkOutageDetector = new NetworkOutageDetector({
             windowMs: this.#options.networkOutageWindowMs,
             errorThreshold: this.#options.networkOutageErrorThreshold,
             hostThreshold: this.#options.networkOutageHostThreshold,
         });
+        this.#dedupeCapTracker = new DedupeCapTracker({ cap: this.#options.dedupeCap ?? 0, mapCap: this.#options.dedupeMapCap }, this.#options.preloadedStickyShapeKeys);
         this.#robotsChecker = new RobotsChecker(this.#options.userAgent, !this.#options.ignoreRobots);
         for (const urlStr of this.#options.roots) {
             const url = parseUrl(urlStr, this.#options);
@@ -231,6 +279,20 @@ class Crawler extends EventEmitter {
     abort() {
         this.#abortController.abort();
     }
+    /**
+     * Per-shape count of anchors the dedupe-cap enqueue gates rejected after
+     * that shape capped (opt-in `--dedupe-cap`). Read by
+     * `CrawlerOrchestrator` at `crawlEnd` to finalize each
+     * `dedupe_cap_events.rejected_count` exactly once — rejections are
+     * accumulated in memory rather than written to the archive per-rejection
+     * to avoid write amplification (a capped trap can generate an unbounded
+     * number of rejected anchors).
+     * @returns A snapshot of the per-shape rejection counts. Empty when
+     *   `--dedupe-cap` was not enabled or no shape has capped yet.
+     */
+    getDedupeCapRejections() {
+        return this.#dedupeCapRejectionCounts;
+    }
     /**
      * Retrieve the list of Chromium process IDs that are still running.
      *
@@ -493,21 +555,92 @@ class Crawler extends EventEmitter {
      * @param enqueue - Callback to enqueue newly discovered URLs into the dealer
      *   queue, prioritising likely-HTML URLs to the front (see {@link partitionUrlsByHtml}).
      *   Accepts a batch so a group of URLs (e.g. predicted pagination) keeps its order.
-     * @param paginationState - Mutable state for predicted pagination cascade prevention
-     * @param paginationState.lastPushedUrl
-     * @param paginationState.lastPushedWasPredicted
      * @param concurrency - Current concurrency level, used to determine predicted URL count
+     * @param precomputedBodyHash - This page's body hash, if the caller already
+     *   computed it (the predicted-content-duplicate check, A-3, computes it for
+     *   every predicted page regardless of `--dedupe-cap`) — reused for the
+     *   dedupe-cap observation below instead of hashing the same html twice.
      */
-    #handleResult(result, url, enqueue, paginationState, concurrency) {
+    #handleResult(result, url, enqueue, concurrency, precomputedBodyHash) {
         switch (result.type) {
             case 'success': {
                 if (!result.pageData)
                     break;
+                // Scoped to this one page's anchor list (fresh per `#handleResult`
+                // call, not shared across pages): pagination-pattern detection
+                // compares consecutive anchors as they are discovered by
+                // `processAnchors`'s single synchronous loop below, so "consecutive"
+                // must mean "adjacent in this document", not "adjacent in whatever
+                // order the crawl's workers happened to finish". Sharing this state
+                // across pages/workers let `step` be computed from two unrelated
+                // URLs, compounding across rounds until a `/news/date/{year}/`
+                // pager's predicted token overflowed into scientific notation
+                // (`1.7715854126052197e+120`, observed in production).
+                const paginationState = {
+                    lastPushedUrl: null,
+                    lastPushedWasPredicted: false,
+                };
+                // Feed this page's own signature into the same-cluster tracker
+                // (opt-in via `--dedupe-cap`). This is deliberately separate
+                // from the enqueue gates below: gating decides whether to
+                // admit a not-yet-scraped anchor based on shape alone; this
+                // observes the page that was JUST scraped, using its actual
+                // meta/body content. External and metadata-only pages carry no
+                // useful signal for this feature and are skipped, matching the
+                // signature-scope exclusions in `computeMetaSignature`'s design.
+                if (this.#options.dedupeCap !== null &&
+                    !result.pageData.isExternal &&
+                    !this.#linkList.isMetadataOnly(result.pageData.url.withoutHash) &&
+                    result.pageData.html.length > 0) {
+                    const shapeKey = computeShapeKey(result.pageData.url.withoutHashAndAuth);
+                    const metaSig = computeMetaSignature(result.pageData.meta);
+                    if (shapeKey && metaSig) {
+                        const bodyHash = precomputedBodyHash ?? computeBodyHash(result.pageData.html);
+                        const ogUrlMismatch = resolveOgUrlMismatch(result.pageData.meta, result.pageData.url.href);
+                        const event = this.#dedupeCapTracker.observe({
+                            shapeKey,
+                            metaSig,
+                            bodyHash,
+                            ogUrlMismatch,
+                            url: result.pageData.url.href,
+                        });
+                        if (event) {
+                            void this.emit('dedupeCap', event);
+                        }
+                    }
+                }
                 handleScrapeEnd(result.pageData, this.#linkList, this.#scope, this.#options, (newUrl, opts) => {
+                    // Gate 1: blocks real anchors discovered on this page whose
+                    // shape is already confirmed as a trap. This does NOT cover
+                    // predicted URLs — `generatePredictedUrls`'s output is
+                    // pushed directly below (`this.#linkList.add(specUrl, ...)`),
+                    // bypassing this closure entirely — so the predicted-URL
+                    // generation site below has its own equivalent check
+                    // (`shapeIsStopped`, combined with `#predictedShapeStopped`).
+                    // External anchors are out of scope for the cap (issue #208:
+                    // "cap 適用は internal only"), enforced by the scope check
+                    // below. Deliberately NOT also excluding `opts?.metadataOnly`
+                    // (unlike the tracker's observation side, which does skip
+                    // metadata-only pages — they carry no reliable signature): with
+                    // `--recursive=false`, `handle-scrape-end.ts` marks EVERY anchor
+                    // metadata-only, internal or not, so excluding them here would
+                    // silently disable `--dedupe-cap` for anchor discovery whenever
+                    // `--recursive=false` is set — while gate 2 (the JS-redirect
+                    // direct enqueue below) has no such exclusion and would still
+                    // cap the very same shape, an inconsistency between the two
+                    // discovery paths.
+                    if (this.#options.dedupeCap !== null &&
+                        findScopeEntry(newUrl, this.#scope, this.#options) !== null) {
+                        const gateShapeKey = computeShapeKey(newUrl.withoutHashAndAuth);
+                        if (gateShapeKey && this.#dedupeCapTracker.isCapped(gateShapeKey)) {
+                            this.#recordDedupeCapRejection(gateShapeKey);
+                            return;
+                        }
+                    }
                     this.#linkList.add(newUrl, opts);
                     void enqueue(newUrl);
                     // Predicted pagination detection
-                    if (!paginationState || !concurrency)
+                    if (!concurrency)
                         return;
                     // metadataOnly / external: update tracking but skip pattern detection
                     if (opts?.metadataOnly ||
@@ -521,22 +654,37 @@ class Crawler extends EventEmitter {
                         !paginationState.lastPushedWasPredicted) {
                         const pattern = detectPaginationPattern(paginationState.lastPushedUrl, newUrl.withoutHashAndAuth);
                         if (pattern) {
-                            const urls = generatePredictedUrls(pattern, newUrl.withoutHashAndAuth, concurrency);
-                            const specUrls = [];
-                            for (const specUrlStr of urls) {
-                                const specUrl = parseUrl(specUrlStr, this.#options);
-                                if (specUrl) {
-                                    this.#linkList.add(specUrl, { predicted: true });
-                                    specUrls.push(specUrl);
+                            // Stop generating further predicted URLs for this shape
+                            // once EITHER confirmation mechanism has fired — the
+                            // always-on content-duplication check
+                            // (`#predictedShapeStopped`), or the opt-in
+                            // `--dedupe-cap` tracker (`#dedupeCapTracker.isCapped`,
+                            // only consulted when the flag is set). Falls through to
+                            // the plain (non-predicted) bookkeeping below instead of
+                            // returning, since the anchor itself is still real.
+                            const shapeKey = computeShapeKey(newUrl.withoutHashAndAuth);
+                            const shapeIsStopped = shapeKey !== null &&
+                                (this.#predictedShapeStopped.has(shapeKey) ||
+                                    (this.#options.dedupeCap !== null &&
+                                        this.#dedupeCapTracker.isCapped(shapeKey)));
+                            if (!shapeIsStopped) {
+                                const urls = generatePredictedUrls(pattern, newUrl.withoutHashAndAuth, concurrency);
+                                const specUrls = [];
+                                for (const specUrlStr of urls) {
+                                    const specUrl = parseUrl(specUrlStr, this.#options);
+                                    if (specUrl) {
+                                        this.#linkList.add(specUrl, { predicted: true });
+                                        specUrls.push(specUrl);
+                                    }
                                 }
+                                // Enqueue as one batch so ascending page order is kept
+                                // at the front of the queue (see enqueue in #runDeal).
+                                if (specUrls.length > 0)
+                                    void enqueue(...specUrls);
+                                paginationState.lastPushedUrl = newUrl.withoutHashAndAuth;
+                                paginationState.lastPushedWasPredicted = true;
+                                return;
                             }
-                            // Enqueue as one batch so ascending page order is kept
-                            // at the front of the queue (see enqueue in #runDeal).
-                            if (specUrls.length > 0)
-                                void enqueue(...specUrls);
-                            paginationState.lastPushedUrl = newUrl.withoutHashAndAuth;
-                            paginationState.lastPushedWasPredicted = true;
-                            return;
                         }
                     }
                     paginationState.lastPushedUrl = newUrl.withoutHashAndAuth;
@@ -630,6 +778,19 @@ class Crawler extends EventEmitter {
             window: { startedAt, endedAt },
         });
     }
+    /**
+     * Increments {@link #dedupeCapRejectionCounts} for one shape. Scoped to
+     * the two concrete enqueue-time rejections (a real anchor or a
+     * JS-redirect destination that was discovered but blocked) — it does
+     * NOT count predicted URLs that were never generated at all because
+     * their shape was already stopped (see the `shapeIsStopped` check in
+     * {@link #handleResult}), since nothing concrete existed there to
+     * reject.
+     * @param shapeKey - The capped shape a rejection is being recorded for.
+     */
+    #recordDedupeCapRejection(shapeKey) {
+        this.#dedupeCapRejectionCounts.set(shapeKey, (this.#dedupeCapRejectionCounts.get(shapeKey) ?? 0) + 1);
+    }
     /**
      * Feed one observed network-layer error into
      * {@link #networkOutageDetector} and hand off to
@@ -746,11 +907,6 @@ class Crawler extends EventEmitter {
         const concurrency = this.#options.parallels
             ? Math.max(this.#options.parallels, 1)
             : _a.MAX_PROCESS_LENGTH;
-        // Predicted pagination state
-        const paginationState = {
-            lastPushedUrl: null,
-            lastPushedWasPredicted: false,
-        };
         await deal(initialUrls, (url, update, _index, setLineHeader, push, unshift) => {
             const matchedScope = findScopeEntry(url, this.#scope, this.#options);
             const isExternal = matchedScope === null;
@@ -805,6 +961,11 @@ class Crawler extends EventEmitter {
                 const markBrowserScrape = () => {
                     renderedInBrowser = true;
                 };
+                // Set by the predicted-content-duplicate check below (A-3) when it
+                // computes this page's body hash, so `#handleResult`'s dedupe-cap
+                // observation (also gated on this page's html) can reuse it instead
+                // of hashing the same html a second time.
+                let precomputedBodyHash = null;
                 try {
                     const robotsAllowed = await this.#robotsChecker.isAllowed(url);
                     if (!robotsAllowed) {
@@ -882,8 +1043,25 @@ class Crawler extends EventEmitter {
                             if (destination) {
                                 const destinationUrl = parseUrl(destination, this.#options);
                                 if (destinationUrl) {
-                                    this.#linkList.add(destinationUrl);
-                                    void enqueue(destinationUrl);
+                                    // Gate 2: this direct enqueue does not go through
+                                    // `#handleResult`'s addUrl closure (gate 1), so it needs
+                                    // its own same-cluster-cap check — a JS-redirect trap
+                                    // that advances a parameter via `location.replace()`
+                                    // would otherwise keep re-entering the queue here.
+                                    const gateShapeKey = computeShapeKey(destinationUrl.withoutHashAndAuth);
+                                    const isCapped = this.#options.dedupeCap !== null &&
+                                        gateShapeKey !== null &&
+                                        findScopeEntry(destinationUrl, this.#scope, this.#options) !==
+                                            null &&
+                                        this.#dedupeCapTracker.isCapped(gateShapeKey);
+                                    if (isCapped) {
+                                        if (gateShapeKey)
+                                            this.#recordDedupeCapRejection(gateShapeKey);
+                                    }
+                                    else {
+                                        this.#linkList.add(destinationUrl);
+                                        void enqueue(destinationUrl);
+                                    }
                                 }
                                 else {
                                     // `deriveJsRedirectTarget` already canonicalises
@@ -926,6 +1104,32 @@ class Crawler extends EventEmitter {
                         log(c.dim('Predicted (discarded)'));
                         return;
                     }
+                    // Discard a predicted URL whose rendered body is a
+                    // byte-for-byte duplicate of the previous predicted page of the
+                    // same shape, and stop generating further predictions for that
+                    // shape (checked above, in the pagination-pattern branch). This
+                    // is the always-on backstop against a site that returns 2xx for
+                    // any extrapolated token but ignores it entirely (e.g. always
+                    // serving the same "no results" template) — `shouldDiscardPredicted`
+                    // alone cannot see this, since it only inspects HTTP status.
+                    if (isPredicted &&
+                        result.type === 'success' &&
+                        result.pageData &&
+                        result.pageData.html.length > 0) {
+                        const shapeKey = computeShapeKey(url.withoutHashAndAuth);
+                        if (shapeKey) {
+                            const bodyHash = computeBodyHash(result.pageData.html);
+                            precomputedBodyHash = bodyHash;
+                            const lastBodyHash = this.#predictedShapeBodyHashes.get(shapeKey) ?? null;
+                            if (isPredictedContentDuplicate(bodyHash, lastBodyHash)) {
+                                this.#predictedShapeStopped.add(shapeKey);
+                                handleIgnoreAndSkip(url, this.#linkList, this.#scope, this.#options);
+                                log(c.dim('Predicted (content duplicate, discarded)'));
+                                return;
+                            }
+                            this.#predictedShapeBodyHashes.set(shapeKey, bodyHash);
+                        }
+                    }
                     // Count only after discard check: rendered HTML pages that
                     // will be persisted to the archive. Launch failures bypass
                     // this point via the catch block; discarded predicted URLs
@@ -934,7 +1138,7 @@ class Crawler extends EventEmitter {
                         pagesScraped++;
                     }
                     log('Saving results%dots%');
-                    this.#handleResult(result, url, enqueue, paginationState, concurrency);
+                    this.#handleResult(result, url, enqueue, concurrency, precomputedBodyHash);
                     const parentSource = await this.#resolveParentSource(url);
                     this.#handleResources(result.resources, parentSource);
                     this.#handleConsoleLogs(result.consoleLogs, url, result.pageData?.redirectPaths ?? []);
@@ -1625,9 +1829,15 @@ class Crawler extends EventEmitter {
             // Verified by `scope-auth-leak.e2e.ts`: removing either piece
             // causes that test to fail (without auth → main 401 hangs;
             // without strip → scope cred leaks to off-scope sub-resource).
+            //
+            // The ExURL fields keep the WHATWG percent-encoded form, but
+            // `page.authenticate` sends its arguments verbatim — decode
+            // first or a password containing `[`/`]`/`{`/`}`/`=` etc.
+            // authenticates with the wrong literal (see
+            // `decode-auth-credential.ts`).
             await page.authenticate({
-                username: url.username ?? '',
-                password: url.password ?? '',
+                username: decodeAuthCredential(url.username),
+                password: decodeAuthCredential(url.password),
             });
             // Re-parse from `withoutHashAndAuth` rather than mutating the
             // re-parsed `url.href` object: ExURL pre-computes `href`,

package/lib/crawler/decode-auth-credential.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Decode a percent-encoded userinfo credential (username or password)
+ * parsed out of a URL into the literal string the origin server expects.
+ *
+ * The WHATWG URL parser percent-encodes characters outside the userinfo
+ * set (`[`, `]`, `{`, `}`, `=`, `:`, `@`, non-ASCII, …) and keeps the
+ * `username` / `password` fields in that encoded form. Consumers that
+ * forward credentials out-of-band — `page.authenticate()` for the
+ * browser session, the `auth` request option for the HEAD pre-flight —
+ * must send the decoded literal, or any credential containing such a
+ * character silently authenticates with the wrong string and the server
+ * answers 401. Node's own `urlToOptions` applies the same
+ * `decodeURIComponent` step for `http.request(url)`.
+ *
+ * A malformed sequence (a literal `%` the parser left untouched, e.g. a
+ * user typing `pa%ssword` without encoding it) would make
+ * `decodeURIComponent` throw, so the raw value is returned as a
+ * fallback — identical to the pre-decode behavior for that input.
+ * @param value - The raw (possibly percent-encoded) credential field, or
+ *   `null` when the URL carries no userinfo.
+ * @returns The decoded credential, or an empty string for `null` input.
+ * @example
+ * ```ts
+ * const url = parseUrl('https://user:pa%5Dss%7Bword%3D@example.com/')!;
+ * decodeAuthCredential(url.password); // => 'pa]ss{word='
+ * decodeAuthCredential(null); // => ''
+ * ```
+ */
+export declare function decodeAuthCredential(value: string | null): string;

package/lib/crawler/decode-auth-credential.js

@@ -0,0 +1,39 @@
+/**
+ * Decode a percent-encoded userinfo credential (username or password)
+ * parsed out of a URL into the literal string the origin server expects.
+ *
+ * The WHATWG URL parser percent-encodes characters outside the userinfo
+ * set (`[`, `]`, `{`, `}`, `=`, `:`, `@`, non-ASCII, …) and keeps the
+ * `username` / `password` fields in that encoded form. Consumers that
+ * forward credentials out-of-band — `page.authenticate()` for the
+ * browser session, the `auth` request option for the HEAD pre-flight —
+ * must send the decoded literal, or any credential containing such a
+ * character silently authenticates with the wrong string and the server
+ * answers 401. Node's own `urlToOptions` applies the same
+ * `decodeURIComponent` step for `http.request(url)`.
+ *
+ * A malformed sequence (a literal `%` the parser left untouched, e.g. a
+ * user typing `pa%ssword` without encoding it) would make
+ * `decodeURIComponent` throw, so the raw value is returned as a
+ * fallback — identical to the pre-decode behavior for that input.
+ * @param value - The raw (possibly percent-encoded) credential field, or
+ *   `null` when the URL carries no userinfo.
+ * @returns The decoded credential, or an empty string for `null` input.
+ * @example
+ * ```ts
+ * const url = parseUrl('https://user:pa%5Dss%7Bword%3D@example.com/')!;
+ * decodeAuthCredential(url.password); // => 'pa]ss{word='
+ * decodeAuthCredential(null); // => ''
+ * ```
+ */
+export function decodeAuthCredential(value) {
+    if (!value) {
+        return '';
+    }
+    try {
+        return decodeURIComponent(value);
+    }
+    catch {
+        return value;
+    }
+}