@nitpicker/crawler 0.14.0 → 0.16.0

package/lib/crawler/crawler.js

@@ -9,17 +9,29 @@ import { retryCall } from '@d-zero/shared/retry';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
 import c from 'ansi-colors';
 import pkg from '../../package.json' with { type: 'json' };
+import { computeBodyHash } from '../archive/body-hash/compute-body-hash.js';
 import { classifyErrorKind } from '../classify-error-kind.js';
 import { crawlerLog } from '../debug.js';
 import { buildJsRedirectEdge } from './build-js-redirect-edge.js';
 import { buildRedirectEvent } from './build-redirect-event.js';
 import { captureImageDomPaths } from './capture-image-dom-paths.js';
+import { chooseProbeHost } from './choose-probe-host.js';
 import { createChangePhaseHandler } from './create-change-phase-handler.js';
+import { decodeAuthCredential } from './decode-auth-credential.js';
+import { computeMetaSignature } from './dedupe/compute-meta-signature.js';
+import { computeShapeKey } from './dedupe/compute-shape-key.js';
+import DedupeCapTracker from './dedupe/dedupe-cap-tracker.js';
+import { isPredictedContentDuplicate } from './dedupe/is-predicted-content-duplicate.js';
+import { resolveOgUrlMismatch } from './dedupe/resolve-og-url-mismatch.js';
 import { derivePageSource } from './derive-page-source.js';
+import { destinationCache } from './destination-cache.js';
 import { detectPaginationPattern } from './detect-pagination-pattern.js';
+import { dnsBurnedHostBurnTimestamps } from './dns-burned-host-burn-timestamps.js';
 import { dnsBurnedHostCache } from './dns-burned-host-cache.js';
 import { dnsBurnedHostShortCircuitCounter } from './dns-burned-host-short-circuit-counter.js';
 import { drainPhaseErrors } from './drain-phase-errors.js';
+import { evictNetworkClassifiedDestinationCacheEntries } from './evict-network-classified-destination-cache-entries.js';
+import { evictOutageTaintedDnsBurns } from './evict-outage-tainted-dns-burns.js';
 import { fetchDestination } from './fetch-destination.js';
 import { findScopeEntry } from './find-scope-entry.js';
 import { formatCrawlProgress } from './format-crawl-progress.js';
@@ -35,9 +47,12 @@ import { isPuppeteerFallbackCandidate } from './is-puppeteer-fallback-candidate.
 import LinkList from './link-list.js';
 import { linkToPageData } from './link-to-page-data.js';
 import { logUndrainedPhaseErrors } from './log-undrained-phase-errors.js';
+import NetworkGate from './network-gate.js';
+import NetworkOutageDetector from './network-outage-detector.js';
 import { partitionUrlsByHtml } from './partition-urls-by-html.js';
 import { planSubResourceEmits } from './plan-sub-resource-emits.js';
 import { PreloadShortCircuitError } from './preload-short-circuit-error.js';
+import { probeNetwork } from './probe-network.js';
 import { protocolAgnosticKey } from './protocol-agnostic-key.js';
 import { redirectDestKey } from './redirect-dest-key.js';
 import { resourceToPageData } from './resource-to-page-data.js';
@@ -56,6 +71,16 @@ import { shouldSkipUrl } from './should-skip-url.js';
  * stay on the final (longest) timeout for any additional attempts.
  */
 const HEAD_TIMEOUT_ESCALATION_MS = [10_000, 30_000, 60_000];
+/** Default {@link CrawlerOptions.networkOutageWindowMs}. */
+const DEFAULT_NETWORK_OUTAGE_WINDOW_MS = 10_000;
+/** Default {@link CrawlerOptions.networkOutageErrorThreshold}. */
+const DEFAULT_NETWORK_OUTAGE_ERROR_THRESHOLD = 5;
+/** Default {@link CrawlerOptions.networkOutageHostThreshold}. */
+const DEFAULT_NETWORK_OUTAGE_HOST_THRESHOLD = 2;
+/** Default {@link CrawlerOptions.networkOutageProbeIntervalMs}. */
+const DEFAULT_NETWORK_OUTAGE_PROBE_INTERVAL_MS = 10_000;
+/** Default {@link CrawlerOptions.dedupeMapCap}. */
+const DEFAULT_DEDUPE_MAP_CAP = 100_000;
 /**
  * The core crawler engine that discovers and scrapes web pages.
  *
@@ -70,10 +95,59 @@ const HEAD_TIMEOUT_ESCALATION_MS = [10_000, 30_000, 60_000];
 class Crawler extends EventEmitter {
     /** Controller used to cancel the deal-based crawl via its AbortSignal. */
     #abortController = new AbortController();
+    /**
+     * Per-shape count of anchors rejected by the dedupe-cap enqueue gates
+     * after that shape capped. Read by {@link getDedupeCapRejections} at
+     * `crawlEnd` so the orchestrator can finalize each
+     * `dedupe_cap_events.rejected_count` exactly once (see
+     * `Crawler#getDedupeCapRejections`'s JSDoc for why this is not written
+     * to the archive incrementally).
+     */
+    #dedupeCapRejectionCounts = new Map();
+    /**
+     * Opt-in (`--dedupe-cap`) same-cluster soft cap. Always constructed
+     * (Misra-Gries state stays empty when {@link CrawlerOptions.dedupeCap} is
+     * `null`), gated on by `#options.dedupeCap !== null` at each call site
+     * rather than being conditionally `undefined`, so the two enqueue gates
+     * and the observation call in {@link #handleResult} do not need to
+     * null-check a class field.
+     */
+    #dedupeCapTracker;
     /** Tracks discovered URLs, their scrape status, and deduplication. */
     #linkList = new LinkList();
+    /**
+     * Gate every worker callback awaits before doing network work (see the
+     * worker body inside {@link #runDeal}). Open by default; closed by
+     * {@link #handleOutageSuspect} once a recovery probe confirms a suspect
+     * outage, reopened once a later probe succeeds. Re-opened defensively at
+     * the start of {@link #runDeal} (a no-op if already open) so a fresh
+     * session never inherits a closed gate from a prior anomalous one.
+     */
+    #networkGate = new NetworkGate();
+    /**
+     * Sliding-window detector for "the operator's own network, not the
+     * target sites, looks like it is down". Fed from {@link #sendHeadRequest}'s
+     * `onWait` / `onGiveUp`; a non-null {@link OutageSuspect} triggers
+     * {@link #handleOutageSuspect}. Reset at the start of {@link #runDeal}.
+     * Assigned in the constructor (not a field initializer) because it
+     * needs `this.#options`'s network-outage tunables.
+     */
+    #networkOutageDetector;
     /** Merged crawler configuration (user overrides + defaults). */
     #options;
+    /**
+     * Synchronous claim flag guarding the async gap between "a suspect
+     * outage arrived" and "the confirming probe settled" in
+     * {@link #handleOutageSuspect}. Without it, two workers whose HEAD
+     * requests both exhaust retries in quick succession could each start
+     * their own confirming probe while the gate is still open, and if both
+     * probes fail, both would close the gate and emit
+     * `networkOutageConfirmed` — creating two simultaneously-open
+     * `network_outages` rows for one ongoing outage. Checked and set
+     * synchronously (no `await` between the check and the set), which is
+     * race-free because JS has no thread-level interleaving.
+     */
+    #outageHandlingInProgress = false;
     /**
      * Phase errors observed during {@link Crawler._launchBrowserAndScrape},
      * buffered per URL href so they can be emitted as `pageError` events
@@ -82,6 +156,23 @@ class Crawler extends EventEmitter {
      * `insertPageError`, so the FK resolution via URL always finds the row.
      */
     #pendingPhaseErrors = new Map();
+    /**
+     * Predicted-pagination body-hash tracking (always-on — independent of
+     * the opt-in `--dedupe-cap` tracker). Maps a URL shape key
+     * ({@link computeShapeKey}) to the {@link computeBodyHash} of the most
+     * recently scraped *predicted* page of that shape. Never reset mid-crawl
+     * (persists for the whole session, like {@link #scrapedDestinations}).
+     */
+    #predictedShapeBodyHashes = new Map();
+    /**
+     * Shapes for which {@link #predictedShapeBodyHashes} detected a
+     * content-duplicate predicted page (see {@link isPredictedContentDuplicate}).
+     * Once a shape lands here, no further predicted URLs are generated for it
+     * (checked in {@link #handleResult}'s pagination-pattern branch) — the
+     * cheapest possible way to stop a self-generating trap without needing
+     * the opt-in dedupe-cap machinery.
+     */
+    #predictedShapeStopped = new Set();
     /** Set of resource URLs (without hash) already captured, for deduplication. */
     #resources = new Set();
     /** Number of HTML pages (isTarget=1) scraped in previous sessions, used to seed the progress counter on resume. */
@@ -153,7 +244,21 @@ class Crawler extends EventEmitter {
             lookupResource: options?.lookupResource ?? null,
             lookupPageSource: options?.lookupPageSource ?? null,
             inventoryMode: options?.inventoryMode ?? null,
+            networkOutageWindowMs: options?.networkOutageWindowMs ?? DEFAULT_NETWORK_OUTAGE_WINDOW_MS,
+            networkOutageErrorThreshold: options?.networkOutageErrorThreshold ?? DEFAULT_NETWORK_OUTAGE_ERROR_THRESHOLD,
+            networkOutageHostThreshold: options?.networkOutageHostThreshold ?? DEFAULT_NETWORK_OUTAGE_HOST_THRESHOLD,
+            networkOutageProbeIntervalMs: options?.networkOutageProbeIntervalMs ?? DEFAULT_NETWORK_OUTAGE_PROBE_INTERVAL_MS,
+            networkProbe: options?.networkProbe ?? null,
+            dedupeCap: options?.dedupeCap ?? null,
+            dedupeMapCap: options?.dedupeMapCap ?? DEFAULT_DEDUPE_MAP_CAP,
+            preloadedStickyShapeKeys: options?.preloadedStickyShapeKeys ?? [],
         };
+        this.#networkOutageDetector = new NetworkOutageDetector({
+            windowMs: this.#options.networkOutageWindowMs,
+            errorThreshold: this.#options.networkOutageErrorThreshold,
+            hostThreshold: this.#options.networkOutageHostThreshold,
+        });
+        this.#dedupeCapTracker = new DedupeCapTracker({ cap: this.#options.dedupeCap ?? 0, mapCap: this.#options.dedupeMapCap }, this.#options.preloadedStickyShapeKeys);
         this.#robotsChecker = new RobotsChecker(this.#options.userAgent, !this.#options.ignoreRobots);
         for (const urlStr of this.#options.roots) {
             const url = parseUrl(urlStr, this.#options);
@@ -174,6 +279,20 @@ class Crawler extends EventEmitter {
     abort() {
         this.#abortController.abort();
     }
+    /**
+     * Per-shape count of anchors the dedupe-cap enqueue gates rejected after
+     * that shape capped (opt-in `--dedupe-cap`). Read by
+     * `CrawlerOrchestrator` at `crawlEnd` to finalize each
+     * `dedupe_cap_events.rejected_count` exactly once — rejections are
+     * accumulated in memory rather than written to the archive per-rejection
+     * to avoid write amplification (a capped trap can generate an unbounded
+     * number of rejected anchors).
+     * @returns A snapshot of the per-shape rejection counts. Empty when
+     *   `--dedupe-cap` was not enabled or no shape has capped yet.
+     */
+    getDedupeCapRejections() {
+        return this.#dedupeCapRejectionCounts;
+    }
     /**
      * Retrieve the list of Chromium process IDs that are still running.
      *
@@ -324,6 +443,84 @@ class Crawler extends EventEmitter {
             });
         }
     }
+    /**
+     * Emits captured console messages / page errors for a scrape (issue
+     * #228), skipping the emit entirely when `entries` is empty.
+     *
+     * The empty-skip is deliberate, not an optimization: `replaceConsoleLogs`
+     * replaces a page's rows wholesale (Scoped-Replace, like
+     * `anchor_edges` / `image_items`), so emitting on an empty capture would
+     * wipe out a prior good result on a degraded re-scrape (navigation
+     * timeout, partial render) that legitimately produced nothing this
+     * time — the same trade-off `updatePage` documents for anchors/images.
+     * @param entries - Console log entries captured during the page load.
+     * @param url - The originally-requested URL (not necessarily the page
+     *   that ends up holding the content — see `CrawlerEventTypes.consoleLogs`).
+     * @param redirectPaths - The redirect chain hops captured during fetch,
+     *   in order. Empty when the scrape produced no `pageData` (a
+     *   `'skipped'` / `'error'` result).
+     */
+    #handleConsoleLogs(entries, url, redirectPaths) {
+        if (entries.length === 0) {
+            return;
+        }
+        void this.emit('consoleLogs', {
+            pageUrl: url.withoutHashAndAuth,
+            redirectPaths,
+            entries,
+        });
+    }
+    /**
+     * Confirm a sliding-window suspect via an active probe, and if
+     * confirmed, close {@link #networkGate} and start
+     * {@link #runRecoveryProbeLoop}.
+     *
+     * Guarded by {@link #outageHandlingInProgress} (a synchronous
+     * check-then-set, race-free under JS's single-threaded execution) AND
+     * by `#networkGate.isOpen` — the latter covers the entire duration a
+     * recovery loop is running (no new suspect should re-confirm or
+     * re-probe while one outage is already open), the former covers only
+     * the narrow async gap between "decided to investigate" and "the
+     * confirming probe settled", which the gate-open check alone cannot see
+     * since the gate has not closed yet at that point.
+     * @param suspect - The trigger emitted by {@link NetworkOutageDetector.record}.
+     */
+    async #handleOutageSuspect(suspect) {
+        if (!this.#networkGate.isOpen || this.#outageHandlingInProgress) {
+            return;
+        }
+        this.#outageHandlingInProgress = true;
+        try {
+            // No usable probe target at all (no session successes yet AND no
+            // parseable root URL) — cannot confirm, and cannot ever detect
+            // recovery either, so there is nothing safe to do but leave the
+            // gate open and treat this as inconclusive.
+            const probeHost = chooseProbeHost(this.#successfulHosts, this.#options.roots);
+            if (probeHost === null) {
+                return;
+            }
+            const probe = this.#options.networkProbe ?? probeNetwork;
+            const initiallyReachable = await probe(probeHost);
+            if (initiallyReachable) {
+                // False alarm: the sliding window tripped (e.g. several
+                // unrelated hosts happened to fail close together) but the
+                // probe host answers fine. Leave the gate open.
+                return;
+            }
+            this.#networkGate.close();
+            void this.emit('networkOutageConfirmed', {
+                startedAt: suspect.startedAt,
+                detectedAt: suspect.detectedAt,
+                probeHost,
+                triggerErrorCount: suspect.triggerErrorCount,
+                triggerHostCount: suspect.triggerHostCount,
+            });
+            void this.#runRecoveryProbeLoop(probeHost, suspect.startedAt);
+        }
+        finally {
+            this.#outageHandlingInProgress = false;
+        }
+    }
     /**
      * Processes captured sub-resources from a page scrape, deduplicates them,
      * and emits `response` / `responseReferrers` events for new resources.
@@ -358,21 +555,92 @@ class Crawler extends EventEmitter {
      * @param enqueue - Callback to enqueue newly discovered URLs into the dealer
      *   queue, prioritising likely-HTML URLs to the front (see {@link partitionUrlsByHtml}).
      *   Accepts a batch so a group of URLs (e.g. predicted pagination) keeps its order.
-     * @param paginationState - Mutable state for predicted pagination cascade prevention
-     * @param paginationState.lastPushedUrl
-     * @param paginationState.lastPushedWasPredicted
      * @param concurrency - Current concurrency level, used to determine predicted URL count
+     * @param precomputedBodyHash - This page's body hash, if the caller already
+     *   computed it (the predicted-content-duplicate check, A-3, computes it for
+     *   every predicted page regardless of `--dedupe-cap`) — reused for the
+     *   dedupe-cap observation below instead of hashing the same html twice.
      */
-    #handleResult(result, url, enqueue, paginationState, concurrency) {
+    #handleResult(result, url, enqueue, concurrency, precomputedBodyHash) {
         switch (result.type) {
             case 'success': {
                 if (!result.pageData)
                     break;
+                // Scoped to this one page's anchor list (fresh per `#handleResult`
+                // call, not shared across pages): pagination-pattern detection
+                // compares consecutive anchors as they are discovered by
+                // `processAnchors`'s single synchronous loop below, so "consecutive"
+                // must mean "adjacent in this document", not "adjacent in whatever
+                // order the crawl's workers happened to finish". Sharing this state
+                // across pages/workers let `step` be computed from two unrelated
+                // URLs, compounding across rounds until a `/news/date/{year}/`
+                // pager's predicted token overflowed into scientific notation
+                // (`1.7715854126052197e+120`, observed in production).
+                const paginationState = {
+                    lastPushedUrl: null,
+                    lastPushedWasPredicted: false,
+                };
+                // Feed this page's own signature into the same-cluster tracker
+                // (opt-in via `--dedupe-cap`). This is deliberately separate
+                // from the enqueue gates below: gating decides whether to
+                // admit a not-yet-scraped anchor based on shape alone; this
+                // observes the page that was JUST scraped, using its actual
+                // meta/body content. External and metadata-only pages carry no
+                // useful signal for this feature and are skipped, matching the
+                // signature-scope exclusions in `computeMetaSignature`'s design.
+                if (this.#options.dedupeCap !== null &&
+                    !result.pageData.isExternal &&
+                    !this.#linkList.isMetadataOnly(result.pageData.url.withoutHash) &&
+                    result.pageData.html.length > 0) {
+                    const shapeKey = computeShapeKey(result.pageData.url.withoutHashAndAuth);
+                    const metaSig = computeMetaSignature(result.pageData.meta);
+                    if (shapeKey && metaSig) {
+                        const bodyHash = precomputedBodyHash ?? computeBodyHash(result.pageData.html);
+                        const ogUrlMismatch = resolveOgUrlMismatch(result.pageData.meta, result.pageData.url.href);
+                        const event = this.#dedupeCapTracker.observe({
+                            shapeKey,
+                            metaSig,
+                            bodyHash,
+                            ogUrlMismatch,
+                            url: result.pageData.url.href,
+                        });
+                        if (event) {
+                            void this.emit('dedupeCap', event);
+                        }
+                    }
+                }
                 handleScrapeEnd(result.pageData, this.#linkList, this.#scope, this.#options, (newUrl, opts) => {
+                    // Gate 1: blocks real anchors discovered on this page whose
+                    // shape is already confirmed as a trap. This does NOT cover
+                    // predicted URLs — `generatePredictedUrls`'s output is
+                    // pushed directly below (`this.#linkList.add(specUrl, ...)`),
+                    // bypassing this closure entirely — so the predicted-URL
+                    // generation site below has its own equivalent check
+                    // (`shapeIsStopped`, combined with `#predictedShapeStopped`).
+                    // External anchors are out of scope for the cap (issue #208:
+                    // "cap 適用は internal only"), enforced by the scope check
+                    // below. Deliberately NOT also excluding `opts?.metadataOnly`
+                    // (unlike the tracker's observation side, which does skip
+                    // metadata-only pages — they carry no reliable signature): with
+                    // `--recursive=false`, `handle-scrape-end.ts` marks EVERY anchor
+                    // metadata-only, internal or not, so excluding them here would
+                    // silently disable `--dedupe-cap` for anchor discovery whenever
+                    // `--recursive=false` is set — while gate 2 (the JS-redirect
+                    // direct enqueue below) has no such exclusion and would still
+                    // cap the very same shape, an inconsistency between the two
+                    // discovery paths.
+                    if (this.#options.dedupeCap !== null &&
+                        findScopeEntry(newUrl, this.#scope, this.#options) !== null) {
+                        const gateShapeKey = computeShapeKey(newUrl.withoutHashAndAuth);
+                        if (gateShapeKey && this.#dedupeCapTracker.isCapped(gateShapeKey)) {
+                            this.#recordDedupeCapRejection(gateShapeKey);
+                            return;
+                        }
+                    }
                     this.#linkList.add(newUrl, opts);
                     void enqueue(newUrl);
                     // Predicted pagination detection
-                    if (!paginationState || !concurrency)
+                    if (!concurrency)
                         return;
                     // metadataOnly / external: update tracking but skip pattern detection
                     if (opts?.metadataOnly ||
@@ -386,22 +654,37 @@ class Crawler extends EventEmitter {
                         !paginationState.lastPushedWasPredicted) {
                         const pattern = detectPaginationPattern(paginationState.lastPushedUrl, newUrl.withoutHashAndAuth);
                         if (pattern) {
-                            const urls = generatePredictedUrls(pattern, newUrl.withoutHashAndAuth, concurrency);
-                            const specUrls = [];
-                            for (const specUrlStr of urls) {
-                                const specUrl = parseUrl(specUrlStr, this.#options);
-                                if (specUrl) {
-                                    this.#linkList.add(specUrl, { predicted: true });
-                                    specUrls.push(specUrl);
+                            // Stop generating further predicted URLs for this shape
+                            // once EITHER confirmation mechanism has fired — the
+                            // always-on content-duplication check
+                            // (`#predictedShapeStopped`), or the opt-in
+                            // `--dedupe-cap` tracker (`#dedupeCapTracker.isCapped`,
+                            // only consulted when the flag is set). Falls through to
+                            // the plain (non-predicted) bookkeeping below instead of
+                            // returning, since the anchor itself is still real.
+                            const shapeKey = computeShapeKey(newUrl.withoutHashAndAuth);
+                            const shapeIsStopped = shapeKey !== null &&
+                                (this.#predictedShapeStopped.has(shapeKey) ||
+                                    (this.#options.dedupeCap !== null &&
+                                        this.#dedupeCapTracker.isCapped(shapeKey)));
+                            if (!shapeIsStopped) {
+                                const urls = generatePredictedUrls(pattern, newUrl.withoutHashAndAuth, concurrency);
+                                const specUrls = [];
+                                for (const specUrlStr of urls) {
+                                    const specUrl = parseUrl(specUrlStr, this.#options);
+                                    if (specUrl) {
+                                        this.#linkList.add(specUrl, { predicted: true });
+                                        specUrls.push(specUrl);
+                                    }
                                 }
+                                // Enqueue as one batch so ascending page order is kept
+                                // at the front of the queue (see enqueue in #runDeal).
+                                if (specUrls.length > 0)
+                                    void enqueue(...specUrls);
+                                paginationState.lastPushedUrl = newUrl.withoutHashAndAuth;
+                                paginationState.lastPushedWasPredicted = true;
+                                return;
                             }
-                            // Enqueue as one batch so ascending page order is kept
-                            // at the front of the queue (see enqueue in #runDeal).
-                            if (specUrls.length > 0)
-                                void enqueue(...specUrls);
-                            paginationState.lastPushedUrl = newUrl.withoutHashAndAuth;
-                            paginationState.lastPushedWasPredicted = true;
-                            return;
                         }
                     }
                     paginationState.lastPushedUrl = newUrl.withoutHashAndAuth;
@@ -471,6 +754,67 @@ class Crawler extends EventEmitter {
             }
         }
     }
+    /**
+     * Undo cache damage from the outage window `[startedAt, endedAt]`:
+     * evict `destinationCache` entries whose cached error looks
+     * network-related (any such entry may be stale evidence about the
+     * operator's network, not the target site), and un-burn any
+     * `dnsBurnedHostCache` host THIS session burned during that window
+     * (preload-seeded burns are structurally immune — see
+     * `evict-outage-tainted-dns-burns.ts`).
+     *
+     * Called on every closed→open gate transition, whether triggered by a
+     * successful recovery probe or by an abort — the cached failures are
+     * stale either way, and the eviction itself has no failure mode that
+     * depends on why the gate reopened.
+     * @param startedAt - The outage's `startedAt` (from the triggering `OutageSuspect`).
+     * @param endedAt - The moment the gate is reopening.
+     */
+    #onGateReopened(startedAt, endedAt) {
+        evictNetworkClassifiedDestinationCacheEntries(destinationCache);
+        evictOutageTaintedDnsBurns({
+            cache: dnsBurnedHostCache,
+            burnTimestamps: dnsBurnedHostBurnTimestamps,
+            window: { startedAt, endedAt },
+        });
+    }
+    /**
+     * Increments {@link #dedupeCapRejectionCounts} for one shape. Scoped to
+     * the two concrete enqueue-time rejections (a real anchor or a
+     * JS-redirect destination that was discovered but blocked) — it does
+     * NOT count predicted URLs that were never generated at all because
+     * their shape was already stopped (see the `shapeIsStopped` check in
+     * {@link #handleResult}), since nothing concrete existed there to
+     * reject.
+     * @param shapeKey - The capped shape a rejection is being recorded for.
+     */
+    #recordDedupeCapRejection(shapeKey) {
+        this.#dedupeCapRejectionCounts.set(shapeKey, (this.#dedupeCapRejectionCounts.get(shapeKey) ?? 0) + 1);
+    }
+    /**
+     * Feed one observed network-layer error into
+     * {@link #networkOutageDetector} and hand off to
+     * {@link #handleOutageSuspect} the instant its sliding window trips.
+     *
+     * Called from BOTH `onWait` (every non-final retry attempt) and
+     * `onGiveUp` (the final attempt) inside {@link #sendHeadRequest}, so a
+     * single URL's retry storm contributes every attempt's error, not just
+     * its terminal one — a real network-wide outage is expected to trip the
+     * `hostThreshold` gate from many DIFFERENT hosts' attempts arriving in
+     * the same short window, not from one URL retrying against one host.
+     * @param message - The raw error message to classify.
+     * @param host - Lower-cased hostname the error occurred on.
+     */
+    #recordNetworkError(message, host) {
+        const suspect = this.#networkOutageDetector.record({
+            kind: classifyErrorKind(message),
+            host,
+            at: Date.now(),
+        });
+        if (suspect) {
+            void this.#handleOutageSuspect(suspect);
+        }
+    }
     /**
      * Resolve the source label of the page being scraped so sub-resources
      * captured during its render can inherit the correct lineage label
@@ -541,6 +885,12 @@ class Crawler extends EventEmitter {
         // does not inherit "host alive" claims from a prior run that may have
         // happened on an entirely different network.
         this.#successfulHosts.clear();
+        // Network-outage state is per-crawl too: a sliding window of errors
+        // (or a gate left closed) from a prior run on this same `Crawler`
+        // instance must not leak into a fresh session. `#networkGate.open()`
+        // is a no-op if already open.
+        this.#networkOutageDetector.reset();
+        this.#networkGate.open();
         // external URL の追跡（target は deal の total/done から導出）
         const externalUrls = new Set();
         const externalDoneUrls = new Set();
@@ -557,11 +907,6 @@ class Crawler extends EventEmitter {
         const concurrency = this.#options.parallels
             ? Math.max(this.#options.parallels, 1)
             : _a.MAX_PROCESS_LENGTH;
-        // Predicted pagination state
-        const paginationState = {
-            lastPushedUrl: null,
-            lastPushedWasPredicted: false,
-        };
         await deal(initialUrls, (url, update, _index, setLineHeader, push, unshift) => {
             const matchedScope = findScopeEntry(url, this.#scope, this.#options);
             const isExternal = matchedScope === null;
@@ -588,6 +933,12 @@ class Crawler extends EventEmitter {
                 return Promise.all(ops).then(() => { });
             };
             return async () => {
+                // Pause here, not inside `fetchDestination` or deeper, so a
+                // paused worker shows as a long-running dealer task instead
+                // of requiring any change to `@d-zero/dealer` itself — a
+                // closed gate resolves the instant `#handleOutageSuspect`'s
+                // recovery probe succeeds (see `network-gate.ts`).
+                await this.#networkGate.wait();
                 // Interval delay is handled here instead of by dealer because
                 // DNS-burned hosts must skip the wait entirely. Spending the
                 // per-URL interval on a host the cache already knows is dead
@@ -610,6 +961,11 @@ class Crawler extends EventEmitter {
                 const markBrowserScrape = () => {
                     renderedInBrowser = true;
                 };
+                // Set by the predicted-content-duplicate check below (A-3) when it
+                // computes this page's body hash, so `#handleResult`'s dedupe-cap
+                // observation (also gated on this page's html) can reuse it instead
+                // of hashing the same html a second time.
+                let precomputedBodyHash = null;
                 try {
                     const robotsAllowed = await this.#robotsChecker.isAllowed(url);
                     if (!robotsAllowed) {
@@ -687,8 +1043,25 @@ class Crawler extends EventEmitter {
                             if (destination) {
                                 const destinationUrl = parseUrl(destination, this.#options);
                                 if (destinationUrl) {
-                                    this.#linkList.add(destinationUrl);
-                                    void enqueue(destinationUrl);
+                                    // Gate 2: this direct enqueue does not go through
+                                    // `#handleResult`'s addUrl closure (gate 1), so it needs
+                                    // its own same-cluster-cap check — a JS-redirect trap
+                                    // that advances a parameter via `location.replace()`
+                                    // would otherwise keep re-entering the queue here.
+                                    const gateShapeKey = computeShapeKey(destinationUrl.withoutHashAndAuth);
+                                    const isCapped = this.#options.dedupeCap !== null &&
+                                        gateShapeKey !== null &&
+                                        findScopeEntry(destinationUrl, this.#scope, this.#options) !==
+                                            null &&
+                                        this.#dedupeCapTracker.isCapped(gateShapeKey);
+                                    if (isCapped) {
+                                        if (gateShapeKey)
+                                            this.#recordDedupeCapRejection(gateShapeKey);
+                                    }
+                                    else {
+                                        this.#linkList.add(destinationUrl);
+                                        void enqueue(destinationUrl);
+                                    }
                                 }
                                 else {
                                     // `deriveJsRedirectTarget` already canonicalises
@@ -731,6 +1104,32 @@ class Crawler extends EventEmitter {
                         log(c.dim('Predicted (discarded)'));
                         return;
                     }
+                    // Discard a predicted URL whose rendered body is a
+                    // byte-for-byte duplicate of the previous predicted page of the
+                    // same shape, and stop generating further predictions for that
+                    // shape (checked above, in the pagination-pattern branch). This
+                    // is the always-on backstop against a site that returns 2xx for
+                    // any extrapolated token but ignores it entirely (e.g. always
+                    // serving the same "no results" template) — `shouldDiscardPredicted`
+                    // alone cannot see this, since it only inspects HTTP status.
+                    if (isPredicted &&
+                        result.type === 'success' &&
+                        result.pageData &&
+                        result.pageData.html.length > 0) {
+                        const shapeKey = computeShapeKey(url.withoutHashAndAuth);
+                        if (shapeKey) {
+                            const bodyHash = computeBodyHash(result.pageData.html);
+                            precomputedBodyHash = bodyHash;
+                            const lastBodyHash = this.#predictedShapeBodyHashes.get(shapeKey) ?? null;
+                            if (isPredictedContentDuplicate(bodyHash, lastBodyHash)) {
+                                this.#predictedShapeStopped.add(shapeKey);
+                                handleIgnoreAndSkip(url, this.#linkList, this.#scope, this.#options);
+                                log(c.dim('Predicted (content duplicate, discarded)'));
+                                return;
+                            }
+                            this.#predictedShapeBodyHashes.set(shapeKey, bodyHash);
+                        }
+                    }
                     // Count only after discard check: rendered HTML pages that
                     // will be persisted to the archive. Launch failures bypass
                     // this point via the catch block; discarded predicted URLs
@@ -739,9 +1138,10 @@ class Crawler extends EventEmitter {
                         pagesScraped++;
                     }
                     log('Saving results%dots%');
-                    this.#handleResult(result, url, enqueue, paginationState, concurrency);
+                    this.#handleResult(result, url, enqueue, concurrency, precomputedBodyHash);
                     const parentSource = await this.#resolveParentSource(url);
                     this.#handleResources(result.resources, parentSource);
+                    this.#handleConsoleLogs(result.consoleLogs, url, result.pageData?.redirectPaths ?? []);
                     log(formatResultSummary(result));
                     // Phase errors must be emitted AFTER 'page' / 'externalPage'
                     // so the orchestrator's WriteQueue sees `setPage` before
@@ -815,6 +1215,52 @@ class Crawler extends EventEmitter {
         crawlerLog('Crawl End');
         void this.emit('crawlEnd', {});
     }
+    /**
+     * While {@link #networkGate} is closed, probe every
+     * `networkOutageProbeIntervalMs` until one succeeds, then reopen the
+     * gate and emit `networkOutageRecovered`.
+     *
+     * If the crawl is aborted while this loop is running, the gate is
+     * opened anyway (so any worker stuck on `#networkGate.wait()` can
+     * unblock and `deal()` can resolve) but `networkOutageRecovered` is NOT
+     * emitted — an abort says nothing about whether the network actually
+     * recovered, so the `network_outages` row is deliberately left open for
+     * the next writer session's boot-time finalizer
+     * (`close-stale-open-network-outages.ts`) to resolve. Either way,
+     * {@link #onGateReopened} still runs — the cached failures are stale
+     * regardless of why the gate reopened.
+     * @param probeHost - The hostname to probe, chosen once by
+     *   {@link #handleOutageSuspect} and reused for every attempt in this loop.
+     * @param startedAt - The outage's `startedAt`, forwarded to {@link #onGateReopened}.
+     */
+    async #runRecoveryProbeLoop(probeHost, startedAt) {
+        const probe = this.#options.networkProbe ?? probeNetwork;
+        const bailIfAborted = () => {
+            if (!this.#abortController.signal.aborted) {
+                return false;
+            }
+            this.#networkGate.open();
+            this.#onGateReopened(startedAt, Date.now());
+            return true;
+        };
+        if (bailIfAborted()) {
+            return;
+        }
+        for (;;) {
+            await delay(this.#options.networkOutageProbeIntervalMs);
+            if (bailIfAborted()) {
+                return;
+            }
+            const recovered = await probe(probeHost);
+            if (recovered) {
+                const endedAt = Date.now();
+                this.#networkGate.open();
+                this.#onGateReopened(startedAt, endedAt);
+                void this.emit('networkOutageRecovered', { endedAt });
+                return;
+            }
+        }
+    }
     /**
      * Orchestrates the full scrape pipeline for a single URL.
      *
@@ -875,6 +1321,7 @@ class Crawler extends EventEmitter {
                     type: 'success',
                     pageData: metadataOnly ? { ...pageData, isTarget: false } : pageData,
                     resources: [],
+                    consoleLogs: [],
                 };
             }
         }
@@ -1002,6 +1449,7 @@ class Crawler extends EventEmitter {
                 return {
                     type: 'error',
                     resources: [],
+                    consoleLogs: [],
                     error: {
                         name: error instanceof Error ? error.name : 'Error',
                         message: errorMessage,
@@ -1015,6 +1463,7 @@ class Crawler extends EventEmitter {
             return {
                 type: 'error',
                 resources: [],
+                consoleLogs: [],
                 error: {
                     name: error instanceof Error ? error.name : 'Error',
                     message: errorMessage,
@@ -1062,6 +1511,7 @@ class Crawler extends EventEmitter {
                         type: 'success',
                         pageData: { ...titleResult, isTarget: false },
                         resources: [],
+                        consoleLogs: [],
                     };
                 }
                 catch (error) {
@@ -1072,6 +1522,7 @@ class Crawler extends EventEmitter {
                 type: 'success',
                 pageData: { ...headCheckResult, isTarget: false },
                 resources: [],
+                consoleLogs: [],
             };
         }
         // Non-HTML content — skip browser
@@ -1081,6 +1532,7 @@ class Crawler extends EventEmitter {
                 type: 'success',
                 pageData: headCheckResult,
                 resources: [],
+                consoleLogs: [],
             };
         }
         // HTML or unknown content type — launch browser with preflight result.
@@ -1266,9 +1718,11 @@ class Crawler extends EventEmitter {
             retries: this.#options.retry,
             label: 'HEAD request',
             onWait: (determinedInterval, retryCount, label, error) => {
+                this.#recordNetworkError(error.message, host);
                 update(`${label}: ${error.message} — %countdown(${determinedInterval},fetchHead_${laneIndex}_${retryCount},s)%s (retry #${retryCount + 1})`);
             },
             onGiveUp: (retryCount, error, label) => {
+                this.#recordNetworkError(error.message, host);
                 // Burn the host so subsequent URLs short-circuit — but ONLY
                 // when this is the first time we've ever seen the host fail
                 // in this session. A host that responded earlier is treated
@@ -1284,6 +1738,12 @@ class Crawler extends EventEmitter {
                     successfulHosts: this.#successfulHosts,
                 })) {
                     dnsBurnedHostCache.set(host, 'dns');
+                    // Recorded so a later outage recovery can tell THIS
+                    // burn (possibly outage-caused) apart from a
+                    // preload-seeded one (a cross-session, confirmed-dead
+                    // verdict that must never be undone by an in-session
+                    // recovery) — see `evict-outage-tainted-dns-burns.ts`.
+                    dnsBurnedHostBurnTimestamps.set(host, Date.now());
                 }
                 update(c.red(`${label}: gave up after ${retryCount} retries — ${error.message}`));
             },
@@ -1369,9 +1829,15 @@ class Crawler extends EventEmitter {
             // Verified by `scope-auth-leak.e2e.ts`: removing either piece
             // causes that test to fail (without auth → main 401 hangs;
             // without strip → scope cred leaks to off-scope sub-resource).
+            //
+            // The ExURL fields keep the WHATWG percent-encoded form, but
+            // `page.authenticate` sends its arguments verbatim — decode
+            // first or a password containing `[`/`]`/`{`/`}`/`=` etc.
+            // authenticates with the wrong literal (see
+            // `decode-auth-credential.ts`).
             await page.authenticate({
-                username: url.username ?? '',
-                password: url.password ?? '',
+                username: decodeAuthCredential(url.username),
+                password: decodeAuthCredential(url.password),
             });
             // Re-parse from `withoutHashAndAuth` rather than mutating the
             // re-parsed `url.href` object: ExURL pre-computes `href`,
@@ -1479,6 +1945,7 @@ class Crawler extends EventEmitter {
             return {
                 type: 'error',
                 resources: [],
+                consoleLogs: [],
                 error: {
                     name: error instanceof Error ? error.name : 'Error',
                     message: error instanceof Error ? error.message : String(error),