@nitpicker/crawler 0.14.0 → 0.15.0

package/lib/archive/url-alias/compute-tier-b-alias-key.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Computes the Tier B alias key for `url`: like `computeTierAAliasKey`,
+ * but additionally strips one trailing `/` from the path (the bare root
+ * path `/` is left alone). Two URLs sharing a Tier B key differ only by
+ * trailing-slash presence — a resource-identity signal considerably weaker
+ * than Tier A's (a web server can legitimately serve different content at
+ * `/foo` and `/foo/`), so callers must additionally require a matching
+ * `page_meta.body_hash` before treating a Tier B match as the same page.
+ * @param url - The URL string to compute a key for.
+ * @returns The Tier B key, or `null` if `url` is not a parseable http(s) URL.
+ * @example
+ * ```ts
+ * computeTierBAliasKey('https://example.com/foo');
+ * computeTierBAliasKey('https://example.com/foo/');
+ * // both: 'example.com/foo' -- same key, but the caller must still confirm
+ * // a matching body_hash before merging these as the same page.
+ * ```
+ */
+export declare function computeTierBAliasKey(url: string): string | null;

package/lib/archive/url-alias/compute-tier-b-alias-key.js

@@ -0,0 +1,30 @@
+import { formatAliasKey } from './format-alias-key.js';
+import { parseAliasKeyParts } from './parse-alias-key-parts.js';
+/**
+ * Computes the Tier B alias key for `url`: like `computeTierAAliasKey`,
+ * but additionally strips one trailing `/` from the path (the bare root
+ * path `/` is left alone). Two URLs sharing a Tier B key differ only by
+ * trailing-slash presence — a resource-identity signal considerably weaker
+ * than Tier A's (a web server can legitimately serve different content at
+ * `/foo` and `/foo/`), so callers must additionally require a matching
+ * `page_meta.body_hash` before treating a Tier B match as the same page.
+ * @param url - The URL string to compute a key for.
+ * @returns The Tier B key, or `null` if `url` is not a parseable http(s) URL.
+ * @example
+ * ```ts
+ * computeTierBAliasKey('https://example.com/foo');
+ * computeTierBAliasKey('https://example.com/foo/');
+ * // both: 'example.com/foo' -- same key, but the caller must still confirm
+ * // a matching body_hash before merging these as the same page.
+ * ```
+ */
+export function computeTierBAliasKey(url) {
+    const parts = parseAliasKeyParts(url);
+    if (!parts) {
+        return null;
+    }
+    const path = parts.path.length > 1 && parts.path.endsWith('/')
+        ? parts.path.slice(0, -1)
+        : parts.path;
+    return formatAliasKey({ ...parts, path });
+}

package/lib/archive/url-alias/format-alias-key.d.ts

@@ -0,0 +1,13 @@
+import type { AliasKeyParts } from './types.js';
+/**
+ * Formats parsed alias-key parts into a single string key. `host`/`port`
+ * never contain `/` or `?`, and `path`/`search` always start with those
+ * characters respectively (or `search` is empty), so concatenation without
+ * an explicit delimiter is unambiguous.
+ * @param parts - The parts to format.
+ * @returns The formatted key string.
+ * @example
+ * formatAliasKey({ host: 'example.com', port: '', path: '/about/', search: '' });
+ * // 'example.com/about/'
+ */
+export declare function formatAliasKey(parts: AliasKeyParts): string;

package/lib/archive/url-alias/format-alias-key.js

@@ -0,0 +1,14 @@
+/**
+ * Formats parsed alias-key parts into a single string key. `host`/`port`
+ * never contain `/` or `?`, and `path`/`search` always start with those
+ * characters respectively (or `search` is empty), so concatenation without
+ * an explicit delimiter is unambiguous.
+ * @param parts - The parts to format.
+ * @returns The formatted key string.
+ * @example
+ * formatAliasKey({ host: 'example.com', port: '', path: '/about/', search: '' });
+ * // 'example.com/about/'
+ */
+export function formatAliasKey(parts) {
+    return `${parts.host}${parts.port ? `:${parts.port}` : ''}${parts.path}${parts.search}`;
+}

package/lib/archive/url-alias/parse-alias-key-parts.d.ts

@@ -0,0 +1,15 @@
+import type { AliasKeyParts } from './types.js';
+/**
+ * Parses `url` and extracts the components Tier A / Tier B keys are built
+ * from. Returns `null` for anything that isn't a parseable `http`/`https`
+ * URL — such a row is defensively excluded from alias candidacy entirely
+ * (a non-http(s) URL should not occur for anything the crawler itself
+ * stored, since every `url_refs.url` was already parsed once during
+ * crawling).
+ * @param url - The URL string to parse.
+ * @returns The extracted parts, or `null` if unparseable / not http(s).
+ * @example
+ * parseAliasKeyParts('https://Example.com/about/index.html');
+ * // { host: 'example.com', port: '', path: '/about/', search: '' }
+ */
+export declare function parseAliasKeyParts(url: string): AliasKeyParts | null;

package/lib/archive/url-alias/parse-alias-key-parts.js

@@ -0,0 +1,40 @@
+const INDEX_SUFFIX_PATTERN = /\/index\.\w+$/i;
+/**
+ * Parses `url` and extracts the components Tier A / Tier B keys are built
+ * from. Returns `null` for anything that isn't a parseable `http`/`https`
+ * URL — such a row is defensively excluded from alias candidacy entirely
+ * (a non-http(s) URL should not occur for anything the crawler itself
+ * stored, since every `url_refs.url` was already parsed once during
+ * crawling).
+ * @param url - The URL string to parse.
+ * @returns The extracted parts, or `null` if unparseable / not http(s).
+ * @example
+ * parseAliasKeyParts('https://Example.com/about/index.html');
+ * // { host: 'example.com', port: '', path: '/about/', search: '' }
+ */
+export function parseAliasKeyParts(url) {
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        return null;
+    }
+    if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+        return null;
+    }
+    return {
+        // WHATWG URL already lowercases `hostname` during parsing; the
+        // explicit `.toLowerCase()` documents that intent rather than
+        // relying on it silently.
+        host: parsed.hostname.toLowerCase(),
+        // Empty string when the port is the scheme's default (or
+        // unspecified) — this is why `http://example.com/` and
+        // `https://example.com/` fold to the same key below (both have
+        // `port === ''`) while an explicit non-default port on either
+        // scheme is preserved and kept distinct.
+        port: parsed.port,
+        path: parsed.pathname.replace(INDEX_SUFFIX_PATTERN, '/'),
+        search: parsed.search,
+    };
+}

package/lib/archive/url-alias/types.d.ts

@@ -0,0 +1,7 @@
+/** Components a Tier A / Tier B alias key is built from. */
+export interface AliasKeyParts {
+    host: string;
+    port: string;
+    path: string;
+    search: string;
+}

package/lib/archive/url-alias/types.js

@@ -0,0 +1 @@
+export {};

package/lib/crawler/choose-probe-host.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Pick the hostname a network-outage probe should target.
+ *
+ * Prefers a host that has already answered successfully during this crawl
+ * session (`Crawler.#successfulHosts` — the same "proven alive" evidence
+ * `shouldBurnHost` uses) over a root URL's hostname, and never falls back to
+ * a hardcoded external address (e.g. `1.1.1.1`): the probe should depend
+ * only on infrastructure the crawl is already touching. `ReadonlySet`
+ * iteration order in JS is insertion order, so this deterministically picks
+ * the first host to have succeeded this session.
+ * @param successfulHosts - Hostnames observed to respond in this session.
+ * @param roots - The crawl's configured root URLs (`CrawlerOptions.roots`),
+ *   used as a fallback before any host has succeeded yet.
+ * @returns A probe target hostname, or `null` if neither source yields one
+ *   (e.g. a fresh session with a malformed/empty roots list and no
+ *   successes yet — the caller should treat this as "cannot probe").
+ * @example
+ * ```ts
+ * chooseProbeHost(new Set(['a.example']), ['https://b.example/']); // 'a.example'
+ * chooseProbeHost(new Set(), ['https://b.example/']); // 'b.example'
+ * chooseProbeHost(new Set(), []); // null
+ * ```
+ */
+export declare function chooseProbeHost(successfulHosts: ReadonlySet<string>, roots: readonly string[]): string | null;

package/lib/crawler/choose-probe-host.js

@@ -0,0 +1,38 @@
+/**
+ * Pick the hostname a network-outage probe should target.
+ *
+ * Prefers a host that has already answered successfully during this crawl
+ * session (`Crawler.#successfulHosts` — the same "proven alive" evidence
+ * `shouldBurnHost` uses) over a root URL's hostname, and never falls back to
+ * a hardcoded external address (e.g. `1.1.1.1`): the probe should depend
+ * only on infrastructure the crawl is already touching. `ReadonlySet`
+ * iteration order in JS is insertion order, so this deterministically picks
+ * the first host to have succeeded this session.
+ * @param successfulHosts - Hostnames observed to respond in this session.
+ * @param roots - The crawl's configured root URLs (`CrawlerOptions.roots`),
+ *   used as a fallback before any host has succeeded yet.
+ * @returns A probe target hostname, or `null` if neither source yields one
+ *   (e.g. a fresh session with a malformed/empty roots list and no
+ *   successes yet — the caller should treat this as "cannot probe").
+ * @example
+ * ```ts
+ * chooseProbeHost(new Set(['a.example']), ['https://b.example/']); // 'a.example'
+ * chooseProbeHost(new Set(), ['https://b.example/']); // 'b.example'
+ * chooseProbeHost(new Set(), []); // null
+ * ```
+ */
+export function chooseProbeHost(successfulHosts, roots) {
+    const [firstSuccessfulHost] = successfulHosts;
+    if (firstSuccessfulHost !== undefined) {
+        return firstSuccessfulHost;
+    }
+    for (const root of roots) {
+        try {
+            return new URL(root).hostname;
+        }
+        catch {
+            continue;
+        }
+    }
+    return null;
+}

package/lib/crawler/clear-dns-burned-host-cache.js

@@ -1,3 +1,4 @@
+import { dnsBurnedHostBurnTimestamps } from './dns-burned-host-burn-timestamps.js';
 import { dnsBurnedHostCache } from './dns-burned-host-cache.js';
 import { dnsBurnedHostShortCircuitCounter } from './dns-burned-host-short-circuit-counter.js';
 /**
@@ -8,4 +9,5 @@ import { dnsBurnedHostShortCircuitCounter } from './dns-burned-host-short-circui
 export function clearDnsBurnedHostCache() {
     dnsBurnedHostCache.clear();
     dnsBurnedHostShortCircuitCounter.count = 0;
+    dnsBurnedHostBurnTimestamps.clear();
 }

package/lib/crawler/crawler.js

@@ -14,12 +14,17 @@ import { crawlerLog } from '../debug.js';
 import { buildJsRedirectEdge } from './build-js-redirect-edge.js';
 import { buildRedirectEvent } from './build-redirect-event.js';
 import { captureImageDomPaths } from './capture-image-dom-paths.js';
+import { chooseProbeHost } from './choose-probe-host.js';
 import { createChangePhaseHandler } from './create-change-phase-handler.js';
 import { derivePageSource } from './derive-page-source.js';
+import { destinationCache } from './destination-cache.js';
 import { detectPaginationPattern } from './detect-pagination-pattern.js';
+import { dnsBurnedHostBurnTimestamps } from './dns-burned-host-burn-timestamps.js';
 import { dnsBurnedHostCache } from './dns-burned-host-cache.js';
 import { dnsBurnedHostShortCircuitCounter } from './dns-burned-host-short-circuit-counter.js';
 import { drainPhaseErrors } from './drain-phase-errors.js';
+import { evictNetworkClassifiedDestinationCacheEntries } from './evict-network-classified-destination-cache-entries.js';
+import { evictOutageTaintedDnsBurns } from './evict-outage-tainted-dns-burns.js';
 import { fetchDestination } from './fetch-destination.js';
 import { findScopeEntry } from './find-scope-entry.js';
 import { formatCrawlProgress } from './format-crawl-progress.js';
@@ -35,9 +40,12 @@ import { isPuppeteerFallbackCandidate } from './is-puppeteer-fallback-candidate.
 import LinkList from './link-list.js';
 import { linkToPageData } from './link-to-page-data.js';
 import { logUndrainedPhaseErrors } from './log-undrained-phase-errors.js';
+import NetworkGate from './network-gate.js';
+import NetworkOutageDetector from './network-outage-detector.js';
 import { partitionUrlsByHtml } from './partition-urls-by-html.js';
 import { planSubResourceEmits } from './plan-sub-resource-emits.js';
 import { PreloadShortCircuitError } from './preload-short-circuit-error.js';
+import { probeNetwork } from './probe-network.js';
 import { protocolAgnosticKey } from './protocol-agnostic-key.js';
 import { redirectDestKey } from './redirect-dest-key.js';
 import { resourceToPageData } from './resource-to-page-data.js';
@@ -56,6 +64,14 @@ import { shouldSkipUrl } from './should-skip-url.js';
  * stay on the final (longest) timeout for any additional attempts.
  */
 const HEAD_TIMEOUT_ESCALATION_MS = [10_000, 30_000, 60_000];
+/** Default {@link CrawlerOptions.networkOutageWindowMs}. */
+const DEFAULT_NETWORK_OUTAGE_WINDOW_MS = 10_000;
+/** Default {@link CrawlerOptions.networkOutageErrorThreshold}. */
+const DEFAULT_NETWORK_OUTAGE_ERROR_THRESHOLD = 5;
+/** Default {@link CrawlerOptions.networkOutageHostThreshold}. */
+const DEFAULT_NETWORK_OUTAGE_HOST_THRESHOLD = 2;
+/** Default {@link CrawlerOptions.networkOutageProbeIntervalMs}. */
+const DEFAULT_NETWORK_OUTAGE_PROBE_INTERVAL_MS = 10_000;
 /**
  * The core crawler engine that discovers and scrapes web pages.
  *
@@ -72,8 +88,39 @@ class Crawler extends EventEmitter {
     #abortController = new AbortController();
     /** Tracks discovered URLs, their scrape status, and deduplication. */
     #linkList = new LinkList();
+    /**
+     * Gate every worker callback awaits before doing network work (see the
+     * worker body inside {@link #runDeal}). Open by default; closed by
+     * {@link #handleOutageSuspect} once a recovery probe confirms a suspect
+     * outage, reopened once a later probe succeeds. Re-opened defensively at
+     * the start of {@link #runDeal} (a no-op if already open) so a fresh
+     * session never inherits a closed gate from a prior anomalous one.
+     */
+    #networkGate = new NetworkGate();
+    /**
+     * Sliding-window detector for "the operator's own network, not the
+     * target sites, looks like it is down". Fed from {@link #sendHeadRequest}'s
+     * `onWait` / `onGiveUp`; a non-null {@link OutageSuspect} triggers
+     * {@link #handleOutageSuspect}. Reset at the start of {@link #runDeal}.
+     * Assigned in the constructor (not a field initializer) because it
+     * needs `this.#options`'s network-outage tunables.
+     */
+    #networkOutageDetector;
     /** Merged crawler configuration (user overrides + defaults). */
     #options;
+    /**
+     * Synchronous claim flag guarding the async gap between "a suspect
+     * outage arrived" and "the confirming probe settled" in
+     * {@link #handleOutageSuspect}. Without it, two workers whose HEAD
+     * requests both exhaust retries in quick succession could each start
+     * their own confirming probe while the gate is still open, and if both
+     * probes fail, both would close the gate and emit
+     * `networkOutageConfirmed` — creating two simultaneously-open
+     * `network_outages` rows for one ongoing outage. Checked and set
+     * synchronously (no `await` between the check and the set), which is
+     * race-free because JS has no thread-level interleaving.
+     */
+    #outageHandlingInProgress = false;
     /**
      * Phase errors observed during {@link Crawler._launchBrowserAndScrape},
      * buffered per URL href so they can be emitted as `pageError` events
@@ -153,7 +200,17 @@ class Crawler extends EventEmitter {
             lookupResource: options?.lookupResource ?? null,
             lookupPageSource: options?.lookupPageSource ?? null,
             inventoryMode: options?.inventoryMode ?? null,
+            networkOutageWindowMs: options?.networkOutageWindowMs ?? DEFAULT_NETWORK_OUTAGE_WINDOW_MS,
+            networkOutageErrorThreshold: options?.networkOutageErrorThreshold ?? DEFAULT_NETWORK_OUTAGE_ERROR_THRESHOLD,
+            networkOutageHostThreshold: options?.networkOutageHostThreshold ?? DEFAULT_NETWORK_OUTAGE_HOST_THRESHOLD,
+            networkOutageProbeIntervalMs: options?.networkOutageProbeIntervalMs ?? DEFAULT_NETWORK_OUTAGE_PROBE_INTERVAL_MS,
+            networkProbe: options?.networkProbe ?? null,
         };
+        this.#networkOutageDetector = new NetworkOutageDetector({
+            windowMs: this.#options.networkOutageWindowMs,
+            errorThreshold: this.#options.networkOutageErrorThreshold,
+            hostThreshold: this.#options.networkOutageHostThreshold,
+        });
         this.#robotsChecker = new RobotsChecker(this.#options.userAgent, !this.#options.ignoreRobots);
         for (const urlStr of this.#options.roots) {
             const url = parseUrl(urlStr, this.#options);
@@ -324,6 +381,84 @@ class Crawler extends EventEmitter {
             });
         }
     }
+    /**
+     * Emits captured console messages / page errors for a scrape (issue
+     * #228), skipping the emit entirely when `entries` is empty.
+     *
+     * The empty-skip is deliberate, not an optimization: `replaceConsoleLogs`
+     * replaces a page's rows wholesale (Scoped-Replace, like
+     * `anchor_edges` / `image_items`), so emitting on an empty capture would
+     * wipe out a prior good result on a degraded re-scrape (navigation
+     * timeout, partial render) that legitimately produced nothing this
+     * time — the same trade-off `updatePage` documents for anchors/images.
+     * @param entries - Console log entries captured during the page load.
+     * @param url - The originally-requested URL (not necessarily the page
+     *   that ends up holding the content — see `CrawlerEventTypes.consoleLogs`).
+     * @param redirectPaths - The redirect chain hops captured during fetch,
+     *   in order. Empty when the scrape produced no `pageData` (a
+     *   `'skipped'` / `'error'` result).
+     */
+    #handleConsoleLogs(entries, url, redirectPaths) {
+        if (entries.length === 0) {
+            return;
+        }
+        void this.emit('consoleLogs', {
+            pageUrl: url.withoutHashAndAuth,
+            redirectPaths,
+            entries,
+        });
+    }
+    /**
+     * Confirm a sliding-window suspect via an active probe, and if
+     * confirmed, close {@link #networkGate} and start
+     * {@link #runRecoveryProbeLoop}.
+     *
+     * Guarded by {@link #outageHandlingInProgress} (a synchronous
+     * check-then-set, race-free under JS's single-threaded execution) AND
+     * by `#networkGate.isOpen` — the latter covers the entire duration a
+     * recovery loop is running (no new suspect should re-confirm or
+     * re-probe while one outage is already open), the former covers only
+     * the narrow async gap between "decided to investigate" and "the
+     * confirming probe settled", which the gate-open check alone cannot see
+     * since the gate has not closed yet at that point.
+     * @param suspect - The trigger emitted by {@link NetworkOutageDetector.record}.
+     */
+    async #handleOutageSuspect(suspect) {
+        if (!this.#networkGate.isOpen || this.#outageHandlingInProgress) {
+            return;
+        }
+        this.#outageHandlingInProgress = true;
+        try {
+            // No usable probe target at all (no session successes yet AND no
+            // parseable root URL) — cannot confirm, and cannot ever detect
+            // recovery either, so there is nothing safe to do but leave the
+            // gate open and treat this as inconclusive.
+            const probeHost = chooseProbeHost(this.#successfulHosts, this.#options.roots);
+            if (probeHost === null) {
+                return;
+            }
+            const probe = this.#options.networkProbe ?? probeNetwork;
+            const initiallyReachable = await probe(probeHost);
+            if (initiallyReachable) {
+                // False alarm: the sliding window tripped (e.g. several
+                // unrelated hosts happened to fail close together) but the
+                // probe host answers fine. Leave the gate open.
+                return;
+            }
+            this.#networkGate.close();
+            void this.emit('networkOutageConfirmed', {
+                startedAt: suspect.startedAt,
+                detectedAt: suspect.detectedAt,
+                probeHost,
+                triggerErrorCount: suspect.triggerErrorCount,
+                triggerHostCount: suspect.triggerHostCount,
+            });
+            void this.#runRecoveryProbeLoop(probeHost, suspect.startedAt);
+        }
+        finally {
+            this.#outageHandlingInProgress = false;
+        }
+    }
     /**
      * Processes captured sub-resources from a page scrape, deduplicates them,
      * and emits `response` / `responseReferrers` events for new resources.
@@ -471,6 +606,54 @@ class Crawler extends EventEmitter {
             }
         }
     }
+    /**
+     * Undo cache damage from the outage window `[startedAt, endedAt]`:
+     * evict `destinationCache` entries whose cached error looks
+     * network-related (any such entry may be stale evidence about the
+     * operator's network, not the target site), and un-burn any
+     * `dnsBurnedHostCache` host THIS session burned during that window
+     * (preload-seeded burns are structurally immune — see
+     * `evict-outage-tainted-dns-burns.ts`).
+     *
+     * Called on every closed→open gate transition, whether triggered by a
+     * successful recovery probe or by an abort — the cached failures are
+     * stale either way, and the eviction itself has no failure mode that
+     * depends on why the gate reopened.
+     * @param startedAt - The outage's `startedAt` (from the triggering `OutageSuspect`).
+     * @param endedAt - The moment the gate is reopening.
+     */
+    #onGateReopened(startedAt, endedAt) {
+        evictNetworkClassifiedDestinationCacheEntries(destinationCache);
+        evictOutageTaintedDnsBurns({
+            cache: dnsBurnedHostCache,
+            burnTimestamps: dnsBurnedHostBurnTimestamps,
+            window: { startedAt, endedAt },
+        });
+    }
+    /**
+     * Feed one observed network-layer error into
+     * {@link #networkOutageDetector} and hand off to
+     * {@link #handleOutageSuspect} the instant its sliding window trips.
+     *
+     * Called from BOTH `onWait` (every non-final retry attempt) and
+     * `onGiveUp` (the final attempt) inside {@link #sendHeadRequest}, so a
+     * single URL's retry storm contributes every attempt's error, not just
+     * its terminal one — a real network-wide outage is expected to trip the
+     * `hostThreshold` gate from many DIFFERENT hosts' attempts arriving in
+     * the same short window, not from one URL retrying against one host.
+     * @param message - The raw error message to classify.
+     * @param host - Lower-cased hostname the error occurred on.
+     */
+    #recordNetworkError(message, host) {
+        const suspect = this.#networkOutageDetector.record({
+            kind: classifyErrorKind(message),
+            host,
+            at: Date.now(),
+        });
+        if (suspect) {
+            void this.#handleOutageSuspect(suspect);
+        }
+    }
     /**
      * Resolve the source label of the page being scraped so sub-resources
      * captured during its render can inherit the correct lineage label
@@ -541,6 +724,12 @@ class Crawler extends EventEmitter {
         // does not inherit "host alive" claims from a prior run that may have
         // happened on an entirely different network.
         this.#successfulHosts.clear();
+        // Network-outage state is per-crawl too: a sliding window of errors
+        // (or a gate left closed) from a prior run on this same `Crawler`
+        // instance must not leak into a fresh session. `#networkGate.open()`
+        // is a no-op if already open.
+        this.#networkOutageDetector.reset();
+        this.#networkGate.open();
         // external URL の追跡（target は deal の total/done から導出）
         const externalUrls = new Set();
         const externalDoneUrls = new Set();
@@ -588,6 +777,12 @@ class Crawler extends EventEmitter {
                 return Promise.all(ops).then(() => { });
             };
             return async () => {
+                // Pause here, not inside `fetchDestination` or deeper, so a
+                // paused worker shows as a long-running dealer task instead
+                // of requiring any change to `@d-zero/dealer` itself — a
+                // closed gate resolves the instant `#handleOutageSuspect`'s
+                // recovery probe succeeds (see `network-gate.ts`).
+                await this.#networkGate.wait();
                 // Interval delay is handled here instead of by dealer because
                 // DNS-burned hosts must skip the wait entirely. Spending the
                 // per-URL interval on a host the cache already knows is dead
@@ -742,6 +937,7 @@ class Crawler extends EventEmitter {
                     this.#handleResult(result, url, enqueue, paginationState, concurrency);
                     const parentSource = await this.#resolveParentSource(url);
                     this.#handleResources(result.resources, parentSource);
+                    this.#handleConsoleLogs(result.consoleLogs, url, result.pageData?.redirectPaths ?? []);
                     log(formatResultSummary(result));
                     // Phase errors must be emitted AFTER 'page' / 'externalPage'
                     // so the orchestrator's WriteQueue sees `setPage` before
@@ -815,6 +1011,52 @@ class Crawler extends EventEmitter {
         crawlerLog('Crawl End');
         void this.emit('crawlEnd', {});
     }
+    /**
+     * While {@link #networkGate} is closed, probe every
+     * `networkOutageProbeIntervalMs` until one succeeds, then reopen the
+     * gate and emit `networkOutageRecovered`.
+     *
+     * If the crawl is aborted while this loop is running, the gate is
+     * opened anyway (so any worker stuck on `#networkGate.wait()` can
+     * unblock and `deal()` can resolve) but `networkOutageRecovered` is NOT
+     * emitted — an abort says nothing about whether the network actually
+     * recovered, so the `network_outages` row is deliberately left open for
+     * the next writer session's boot-time finalizer
+     * (`close-stale-open-network-outages.ts`) to resolve. Either way,
+     * {@link #onGateReopened} still runs — the cached failures are stale
+     * regardless of why the gate reopened.
+     * @param probeHost - The hostname to probe, chosen once by
+     *   {@link #handleOutageSuspect} and reused for every attempt in this loop.
+     * @param startedAt - The outage's `startedAt`, forwarded to {@link #onGateReopened}.
+     */
+    async #runRecoveryProbeLoop(probeHost, startedAt) {
+        const probe = this.#options.networkProbe ?? probeNetwork;
+        const bailIfAborted = () => {
+            if (!this.#abortController.signal.aborted) {
+                return false;
+            }
+            this.#networkGate.open();
+            this.#onGateReopened(startedAt, Date.now());
+            return true;
+        };
+        if (bailIfAborted()) {
+            return;
+        }
+        for (;;) {
+            await delay(this.#options.networkOutageProbeIntervalMs);
+            if (bailIfAborted()) {
+                return;
+            }
+            const recovered = await probe(probeHost);
+            if (recovered) {
+                const endedAt = Date.now();
+                this.#networkGate.open();
+                this.#onGateReopened(startedAt, endedAt);
+                void this.emit('networkOutageRecovered', { endedAt });
+                return;
+            }
+        }
+    }
     /**
      * Orchestrates the full scrape pipeline for a single URL.
      *
@@ -875,6 +1117,7 @@ class Crawler extends EventEmitter {
                     type: 'success',
                     pageData: metadataOnly ? { ...pageData, isTarget: false } : pageData,
                     resources: [],
+                    consoleLogs: [],
                 };
             }
         }
@@ -1002,6 +1245,7 @@ class Crawler extends EventEmitter {
                 return {
                     type: 'error',
                     resources: [],
+                    consoleLogs: [],
                     error: {
                         name: error instanceof Error ? error.name : 'Error',
                         message: errorMessage,
@@ -1015,6 +1259,7 @@ class Crawler extends EventEmitter {
             return {
                 type: 'error',
                 resources: [],
+                consoleLogs: [],
                 error: {
                     name: error instanceof Error ? error.name : 'Error',
                     message: errorMessage,
@@ -1062,6 +1307,7 @@ class Crawler extends EventEmitter {
                         type: 'success',
                         pageData: { ...titleResult, isTarget: false },
                         resources: [],
+                        consoleLogs: [],
                     };
                 }
                 catch (error) {
@@ -1072,6 +1318,7 @@ class Crawler extends EventEmitter {
                 type: 'success',
                 pageData: { ...headCheckResult, isTarget: false },
                 resources: [],
+                consoleLogs: [],
             };
         }
         // Non-HTML content — skip browser
@@ -1081,6 +1328,7 @@ class Crawler extends EventEmitter {
                 type: 'success',
                 pageData: headCheckResult,
                 resources: [],
+                consoleLogs: [],
             };
         }
         // HTML or unknown content type — launch browser with preflight result.
@@ -1266,9 +1514,11 @@ class Crawler extends EventEmitter {
             retries: this.#options.retry,
             label: 'HEAD request',
             onWait: (determinedInterval, retryCount, label, error) => {
+                this.#recordNetworkError(error.message, host);
                 update(`${label}: ${error.message} — %countdown(${determinedInterval},fetchHead_${laneIndex}_${retryCount},s)%s (retry #${retryCount + 1})`);
             },
             onGiveUp: (retryCount, error, label) => {
+                this.#recordNetworkError(error.message, host);
                 // Burn the host so subsequent URLs short-circuit — but ONLY
                 // when this is the first time we've ever seen the host fail
                 // in this session. A host that responded earlier is treated
@@ -1284,6 +1534,12 @@ class Crawler extends EventEmitter {
                     successfulHosts: this.#successfulHosts,
                 })) {
                     dnsBurnedHostCache.set(host, 'dns');
+                    // Recorded so a later outage recovery can tell THIS
+                    // burn (possibly outage-caused) apart from a
+                    // preload-seeded one (a cross-session, confirmed-dead
+                    // verdict that must never be undone by an in-session
+                    // recovery) — see `evict-outage-tainted-dns-burns.ts`.
+                    dnsBurnedHostBurnTimestamps.set(host, Date.now());
                 }
                 update(c.red(`${label}: gave up after ${retryCount} retries — ${error.message}`));
             },
@@ -1479,6 +1735,7 @@ class Crawler extends EventEmitter {
             return {
                 type: 'error',
                 resources: [],
+                consoleLogs: [],
                 error: {
                     name: error instanceof Error ? error.name : 'Error',
                     message: error instanceof Error ? error.message : String(error),

package/lib/crawler/dns-burned-host-burn-timestamps.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Companion to `dnsBurnedHostCache`: records the epoch-ms timestamp of each
+ * host burned by THIS session's `#sendHeadRequest` `onGiveUp` path (the
+ * `shouldBurnHost`-gated write in `crawler.ts`).
+ *
+ * Deliberately NOT populated by `#preloadDnsBurnedHostCache`'s seeding from
+ * a previous session's `crawl_errors` — those entries represent hosts
+ * already proven dead across sessions and must never be evicted just
+ * because THIS session's network happened to recover from an outage. Since
+ * `dnsBurnedHostCache`'s value (`ErrorKind`) carries no provenance of its
+ * own (preload-seeded and session-learned burns are indistinguishable by
+ * value alone), this separate timestamp map is what lets
+ * `evict-outage-tainted-dns-burns.ts` tell them apart: only hosts present
+ * HERE are eligible for eviction, and only if their burn timestamp falls
+ * inside the just-recovered outage's window.
+ */
+export declare const dnsBurnedHostBurnTimestamps: Map<string, number>;