@nitpicker/crawler 0.14.0 → 0.15.0

package/lib/crawler/dns-burned-host-burn-timestamps.js

@@ -0,0 +1,17 @@
+/**
+ * Companion to `dnsBurnedHostCache`: records the epoch-ms timestamp of each
+ * host burned by THIS session's `#sendHeadRequest` `onGiveUp` path (the
+ * `shouldBurnHost`-gated write in `crawler.ts`).
+ *
+ * Deliberately NOT populated by `#preloadDnsBurnedHostCache`'s seeding from
+ * a previous session's `crawl_errors` — those entries represent hosts
+ * already proven dead across sessions and must never be evicted just
+ * because THIS session's network happened to recover from an outage. Since
+ * `dnsBurnedHostCache`'s value (`ErrorKind`) carries no provenance of its
+ * own (preload-seeded and session-learned burns are indistinguishable by
+ * value alone), this separate timestamp map is what lets
+ * `evict-outage-tainted-dns-burns.ts` tell them apart: only hosts present
+ * HERE are eligible for eviction, and only if their burn timestamp falls
+ * inside the just-recovered outage's window.
+ */
+export const dnsBurnedHostBurnTimestamps = new Map();

package/lib/crawler/evict-network-classified-destination-cache-entries.d.ts

@@ -0,0 +1,26 @@
+import type { PageData } from '@d-zero/beholder';
+/**
+ * Delete every `destinationCache` entry whose cached value is an `Error`
+ * that classifies as a {@link NETWORK_RELATED_ERROR_KINDS network-related}
+ * kind, leaving successes and site-specific failures (`tls`,
+ * `client-blocked`, `connection-refused`, …) untouched.
+ *
+ * Called by `Crawler` whenever its network gate transitions from closed to
+ * open (whether via a successful recovery probe or an abort), so a URL
+ * that failed with `getaddrinfo ENOTFOUND` only because the operator's own
+ * network was down does not stay wrongly cached as "this host is dead" for
+ * the rest of the session.
+ *
+ * Deliberately does NOT distinguish "cached during THIS specific outage"
+ * from "cached during an earlier blip this session" — any cached error
+ * whose KIND looks network-related is, by definition, potentially stale
+ * evidence about the operator's network rather than the target site, so it
+ * is always safe to re-test after any recovery. The trade-off is a few
+ * redundant HEAD attempts for hosts that were already dead independent of
+ * the outage; the alternative (a stale "outage-tainted" failure verdict
+ * surviving the rest of the session) is strictly worse.
+ * @param cache - The cache to sweep. Takes the `Map` explicitly (rather
+ *   than importing the `destinationCache` singleton directly) so this stays
+ *   unit-testable without touching module-level state.
+ */
+export declare function evictNetworkClassifiedDestinationCacheEntries(cache: Map<string, PageData | Error>): void;

package/lib/crawler/evict-network-classified-destination-cache-entries.js

@@ -0,0 +1,34 @@
+import { classifyErrorKind } from '../classify-error-kind.js';
+import { NETWORK_RELATED_ERROR_KINDS } from '../network-related-error-kinds.js';
+/**
+ * Delete every `destinationCache` entry whose cached value is an `Error`
+ * that classifies as a {@link NETWORK_RELATED_ERROR_KINDS network-related}
+ * kind, leaving successes and site-specific failures (`tls`,
+ * `client-blocked`, `connection-refused`, …) untouched.
+ *
+ * Called by `Crawler` whenever its network gate transitions from closed to
+ * open (whether via a successful recovery probe or an abort), so a URL
+ * that failed with `getaddrinfo ENOTFOUND` only because the operator's own
+ * network was down does not stay wrongly cached as "this host is dead" for
+ * the rest of the session.
+ *
+ * Deliberately does NOT distinguish "cached during THIS specific outage"
+ * from "cached during an earlier blip this session" — any cached error
+ * whose KIND looks network-related is, by definition, potentially stale
+ * evidence about the operator's network rather than the target site, so it
+ * is always safe to re-test after any recovery. The trade-off is a few
+ * redundant HEAD attempts for hosts that were already dead independent of
+ * the outage; the alternative (a stale "outage-tainted" failure verdict
+ * surviving the rest of the session) is strictly worse.
+ * @param cache - The cache to sweep. Takes the `Map` explicitly (rather
+ *   than importing the `destinationCache` singleton directly) so this stays
+ *   unit-testable without touching module-level state.
+ */
+export function evictNetworkClassifiedDestinationCacheEntries(cache) {
+    for (const [key, value] of cache) {
+        if (value instanceof Error &&
+            NETWORK_RELATED_ERROR_KINDS.has(classifyErrorKind(value.message))) {
+            cache.delete(key);
+        }
+    }
+}

package/lib/crawler/evict-outage-tainted-dns-burns.d.ts

@@ -0,0 +1,23 @@
+import type { OutageWindow } from '../is-within-outage-window.js';
+import type { ErrorKind } from '../types.js';
+/**
+ * Undo `dnsBurnedHostCache` burns whose recorded timestamp (see
+ * `dns-burned-host-burn-timestamps.ts`) falls inside `window` — i.e. hosts
+ * THIS session burned because a HEAD request failed with a
+ * `dns`-classified error while (or just before) the operator's own network
+ * was down, not because the host is actually dead.
+ *
+ * Preload-seeded burns are structurally immune: they never appear in
+ * `burnTimestamps` (see that module's docstring), so this function can
+ * never touch them regardless of the window.
+ * @param options - Named parameters.
+ * @param options.cache - `dnsBurnedHostCache` (or a test double with the same shape).
+ * @param options.burnTimestamps - `dnsBurnedHostBurnTimestamps` (or a test double).
+ * @param options.window - The just-resolved outage window (`{ startedAt, endedAt }`)
+ *   to test each burn's timestamp against.
+ */
+export declare function evictOutageTaintedDnsBurns(options: {
+    cache: Map<string, ErrorKind>;
+    burnTimestamps: Map<string, number>;
+    window: OutageWindow;
+}): void;

package/lib/crawler/evict-outage-tainted-dns-burns.js

@@ -0,0 +1,26 @@
+import { isWithinOutageWindow } from '../is-within-outage-window.js';
+/**
+ * Undo `dnsBurnedHostCache` burns whose recorded timestamp (see
+ * `dns-burned-host-burn-timestamps.ts`) falls inside `window` — i.e. hosts
+ * THIS session burned because a HEAD request failed with a
+ * `dns`-classified error while (or just before) the operator's own network
+ * was down, not because the host is actually dead.
+ *
+ * Preload-seeded burns are structurally immune: they never appear in
+ * `burnTimestamps` (see that module's docstring), so this function can
+ * never touch them regardless of the window.
+ * @param options - Named parameters.
+ * @param options.cache - `dnsBurnedHostCache` (or a test double with the same shape).
+ * @param options.burnTimestamps - `dnsBurnedHostBurnTimestamps` (or a test double).
+ * @param options.window - The just-resolved outage window (`{ startedAt, endedAt }`)
+ *   to test each burn's timestamp against.
+ */
+export function evictOutageTaintedDnsBurns(options) {
+    const { cache, burnTimestamps, window } = options;
+    for (const [host, burnedAt] of burnTimestamps) {
+        if (isWithinOutageWindow(burnedAt, [window])) {
+            cache.delete(host);
+            burnTimestamps.delete(host);
+        }
+    }
+}

package/lib/crawler/network-gate.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Level-triggered gate that dealer worker callbacks await before doing any
+ * network work. Achieves "pause the crawl until the network recovers"
+ * without touching `@d-zero/dealer` internals — a closed gate just makes
+ * the worker's task body a long-running `await`, which is all dealer ever
+ * sees.
+ *
+ * **`epoch`** counts outage generations: it increments on every `close()`
+ * transition (open → closed) and holds that value for the remainder of the
+ * closed period AND through the following open period, until the next
+ * `close()`. This is the hook `destinationCache` / `dnsBurnedHostCache`
+ * eviction uses: an entry written while the gate was closed is tagged with
+ * `gate.epoch` at write time; after `open()`, the caller purges entries
+ * tagged with that same epoch value. Because `epoch` only advances on
+ * `close()` (never on `open()`), entries written during ordinary (open)
+ * operation are never mistakenly tagged with a soon-to-be-purged epoch —
+ * only writes that happen while `isOpen` is `false` get tagged at all.
+ */
+export default class NetworkGate {
+    #private;
+    /** `true` when the gate is open (the default). */
+    get isOpen(): boolean;
+    /**
+     * Current outage-generation counter. See the class docstring for how
+     * cache-eviction callers are expected to use this value.
+     */
+    get epoch(): number;
+    /**
+     * Close the gate, blocking future {@link wait} callers until the next
+     * {@link open}. Idempotent — calling `close()` while already closed does
+     * nothing (in particular, it does NOT bump {@link epoch} a second time,
+     * which would otherwise let a still-open outage's cache entries escape
+     * eviction under a stale epoch number).
+     */
+    close(): void;
+    /**
+     * Open the gate, resolving every {@link wait} caller currently pending —
+     * including ones that subscribed after `close()` but before this call.
+     * Idempotent — calling `open()` while already open does nothing.
+     */
+    open(): void;
+    /**
+     * Resolve immediately if the gate is open; otherwise resolve on the next
+     * {@link open}. Multiple concurrent callers all resolve on the same
+     * `open()` call — no awaiter is dropped regardless of when it subscribed
+     * relative to others.
+     */
+    wait(): Promise<void>;
+}

package/lib/crawler/network-gate.js

@@ -0,0 +1,78 @@
+/**
+ * Level-triggered gate that dealer worker callbacks await before doing any
+ * network work. Achieves "pause the crawl until the network recovers"
+ * without touching `@d-zero/dealer` internals — a closed gate just makes
+ * the worker's task body a long-running `await`, which is all dealer ever
+ * sees.
+ *
+ * **`epoch`** counts outage generations: it increments on every `close()`
+ * transition (open → closed) and holds that value for the remainder of the
+ * closed period AND through the following open period, until the next
+ * `close()`. This is the hook `destinationCache` / `dnsBurnedHostCache`
+ * eviction uses: an entry written while the gate was closed is tagged with
+ * `gate.epoch` at write time; after `open()`, the caller purges entries
+ * tagged with that same epoch value. Because `epoch` only advances on
+ * `close()` (never on `open()`), entries written during ordinary (open)
+ * operation are never mistakenly tagged with a soon-to-be-purged epoch —
+ * only writes that happen while `isOpen` is `false` get tagged at all.
+ */
+export default class NetworkGate {
+    #epoch = 0;
+    #open = true;
+    #waiters = [];
+    /** `true` when the gate is open (the default). */
+    get isOpen() {
+        return this.#open;
+    }
+    /**
+     * Current outage-generation counter. See the class docstring for how
+     * cache-eviction callers are expected to use this value.
+     */
+    get epoch() {
+        return this.#epoch;
+    }
+    /**
+     * Close the gate, blocking future {@link wait} callers until the next
+     * {@link open}. Idempotent — calling `close()` while already closed does
+     * nothing (in particular, it does NOT bump {@link epoch} a second time,
+     * which would otherwise let a still-open outage's cache entries escape
+     * eviction under a stale epoch number).
+     */
+    close() {
+        if (!this.#open) {
+            return;
+        }
+        this.#open = false;
+        this.#epoch += 1;
+    }
+    /**
+     * Open the gate, resolving every {@link wait} caller currently pending —
+     * including ones that subscribed after `close()` but before this call.
+     * Idempotent — calling `open()` while already open does nothing.
+     */
+    open() {
+        if (this.#open) {
+            return;
+        }
+        this.#open = true;
+        const waiters = this.#waiters;
+        this.#waiters = [];
+        for (const resolve of waiters) {
+            resolve();
+        }
+    }
+    /**
+     * Resolve immediately if the gate is open; otherwise resolve on the next
+     * {@link open}. Multiple concurrent callers all resolve on the same
+     * `open()` call — no awaiter is dropped regardless of when it subscribed
+     * relative to others.
+     */
+    wait() {
+        if (this.#open) {
+            return Promise.resolve();
+        }
+        return new Promise((resolve) => {
+            this.#waiters.push(resolve);
+        });
+    }
+}

package/lib/crawler/network-outage-detector.d.ts

@@ -0,0 +1,51 @@
+import type { NetworkErrorRecord, NetworkOutageDetectorOptions, OutageSuspect } from './types.js';
+/**
+ * Sliding-window detector for "this looks like the crawl operator's own
+ * network, not the target sites, is having a bad time" — the trigger that
+ * hands off to an active probe (see `probe-network.ts`) before the crawler
+ * commits to pausing.
+ *
+ * Deliberately dependency-free: no timers, no DB, no `Crawler` instance.
+ * Time flows in via {@link NetworkErrorRecord.at} only, which is what makes
+ * window-boundary behaviour testable with exact values instead of
+ * `vi.useFakeTimers()`.
+ * @example
+ * ```ts
+ * const detector = new NetworkOutageDetector({
+ *   windowMs: 10_000,
+ *   errorThreshold: 5,
+ *   hostThreshold: 2,
+ * });
+ * const suspect = detector.record({ kind: 'dns', host: 'a.example', at: Date.now() });
+ * if (suspect) {
+ *   // probe before closing the gate
+ * }
+ * ```
+ */
+export default class NetworkOutageDetector {
+    #private;
+    constructor(options: NetworkOutageDetectorOptions);
+    /**
+     * Record one observed error and check whether it tips the sliding window
+     * over both thresholds.
+     *
+     * Non-network `kind`s (see `NETWORK_RELATED_ERROR_KINDS`) are silently
+     * ignored: they never enter the window and can never contribute to a
+     * trigger, regardless of how many arrive.
+     * @param record - The observed error.
+     * @param record.kind
+     * @param record.host
+     * @param record.at
+     * @returns An {@link OutageSuspect} the instant both thresholds are met,
+     *   or `null` otherwise. On a trigger, the window is cleared so the same
+     *   batch of errors cannot fire a second time on the next call.
+     */
+    record({ kind, host, at }: NetworkErrorRecord): OutageSuspect | null;
+    /**
+     * Clear the sliding window. Called at the start of a fresh crawl session
+     * (`Crawler.#runDeal`) so error observations from a prior session on the
+     * same `Crawler` instance never carry over — mirrors `#successfulHosts
+     * .clear()` / `#scrapedDestinations.clear()`'s per-session reset.
+     */
+    reset(): void;
+}

package/lib/crawler/network-outage-detector.js

@@ -0,0 +1,81 @@
+import { NETWORK_RELATED_ERROR_KINDS } from '../network-related-error-kinds.js';
+/**
+ * Sliding-window detector for "this looks like the crawl operator's own
+ * network, not the target sites, is having a bad time" — the trigger that
+ * hands off to an active probe (see `probe-network.ts`) before the crawler
+ * commits to pausing.
+ *
+ * Deliberately dependency-free: no timers, no DB, no `Crawler` instance.
+ * Time flows in via {@link NetworkErrorRecord.at} only, which is what makes
+ * window-boundary behaviour testable with exact values instead of
+ * `vi.useFakeTimers()`.
+ * @example
+ * ```ts
+ * const detector = new NetworkOutageDetector({
+ *   windowMs: 10_000,
+ *   errorThreshold: 5,
+ *   hostThreshold: 2,
+ * });
+ * const suspect = detector.record({ kind: 'dns', host: 'a.example', at: Date.now() });
+ * if (suspect) {
+ *   // probe before closing the gate
+ * }
+ * ```
+ */
+export default class NetworkOutageDetector {
+    #entries = [];
+    #errorThreshold;
+    #hostThreshold;
+    #windowMs;
+    constructor(options) {
+        this.#windowMs = options.windowMs;
+        this.#errorThreshold = options.errorThreshold;
+        this.#hostThreshold = options.hostThreshold;
+    }
+    /**
+     * Record one observed error and check whether it tips the sliding window
+     * over both thresholds.
+     *
+     * Non-network `kind`s (see `NETWORK_RELATED_ERROR_KINDS`) are silently
+     * ignored: they never enter the window and can never contribute to a
+     * trigger, regardless of how many arrive.
+     * @param record - The observed error.
+     * @param record.kind
+     * @param record.host
+     * @param record.at
+     * @returns An {@link OutageSuspect} the instant both thresholds are met,
+     *   or `null` otherwise. On a trigger, the window is cleared so the same
+     *   batch of errors cannot fire a second time on the next call.
+     */
+    record({ kind, host, at }) {
+        if (!NETWORK_RELATED_ERROR_KINDS.has(kind)) {
+            return null;
+        }
+        const cutoff = at - this.#windowMs;
+        this.#entries = this.#entries.filter((entry) => entry.at >= cutoff);
+        this.#entries.push({ host, at });
+        const distinctHosts = new Set(this.#entries.map((entry) => entry.host)).size;
+        if (this.#entries.length < this.#errorThreshold ||
+            distinctHosts < this.#hostThreshold) {
+            return null;
+        }
+        const startedAt = Math.min(...this.#entries.map((entry) => entry.at));
+        const suspect = {
+            startedAt,
+            detectedAt: at,
+            triggerErrorCount: this.#entries.length,
+            triggerHostCount: distinctHosts,
+        };
+        this.#entries = [];
+        return suspect;
+    }
+    /**
+     * Clear the sliding window. Called at the start of a fresh crawl session
+     * (`Crawler.#runDeal`) so error observations from a prior session on the
+     * same `Crawler` instance never carry over — mirrors `#successfulHosts
+     * .clear()` / `#scrapedDestinations.clear()`'s per-session reset.
+     */
+    reset() {
+        this.#entries = [];
+    }
+}

package/lib/crawler/network-outage-summary-counter.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Session-scoped counter for how many operator-network outages were
+ * confirmed, and their total duration. The orchestrator accumulates into
+ * this at every `networkOutageRecovered` event, reads it at crawl shutdown
+ * to emit a `[network] N outage(s), Ms total` summary line, then zeroes it
+ * for the next session (mirrors `dnsBurnedHostShortCircuitCounter`'s
+ * lifecycle, including its known limitation: two `CrawlerOrchestrator`
+ * crawls running concurrently in the same process would accumulate into and
+ * zero the same shared counter, cross-contaminating each other's summary.
+ * The CLI only ever runs one crawl per process, so this has never been
+ * observed in practice; fixing it would mean threading per-session state
+ * through both counters together, out of scope here).
+ *
+ * Exposed as a mutable object (not a plain `let`) so the counter remains a
+ * single shared reference across crawler / orchestrator imports — `let`
+ * bindings cannot be mutated from another module.
+ */
+export declare const networkOutageSummaryCounter: {
+    confirmedCount: number;
+    totalDurationMs: number;
+};

package/lib/crawler/network-outage-summary-counter.js

@@ -0,0 +1,18 @@
+/**
+ * Session-scoped counter for how many operator-network outages were
+ * confirmed, and their total duration. The orchestrator accumulates into
+ * this at every `networkOutageRecovered` event, reads it at crawl shutdown
+ * to emit a `[network] N outage(s), Ms total` summary line, then zeroes it
+ * for the next session (mirrors `dnsBurnedHostShortCircuitCounter`'s
+ * lifecycle, including its known limitation: two `CrawlerOrchestrator`
+ * crawls running concurrently in the same process would accumulate into and
+ * zero the same shared counter, cross-contaminating each other's summary.
+ * The CLI only ever runs one crawl per process, so this has never been
+ * observed in practice; fixing it would mean threading per-session state
+ * through both counters together, out of scope here).
+ *
+ * Exposed as a mutable object (not a plain `let`) so the counter remains a
+ * single shared reference across crawler / orchestrator imports — `let`
+ * bindings cannot be mutated from another module.
+ */
+export const networkOutageSummaryCounter = { confirmedCount: 0, totalDurationMs: 0 };

package/lib/crawler/probe-network.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Function shape for an injectable network-reachability probe: given a
+ * hostname, resolve to `true` if the network path to it appears healthy,
+ * `false` otherwise. Never rejects — a probe failure is a normal outcome
+ * (that's the whole point of probing), not an exceptional one.
+ *
+ * Injected via `CrawlerOptions.networkProbe` so tests can simulate
+ * confirmed outages and recoveries deterministically (a function that fails
+ * N times then starts succeeding) without touching the real network — the
+ * same injection pattern as `ResourceLookup` / `PageSourceLookup`.
+ */
+export type NetworkProbe = (host: string) => Promise<boolean>;
+/**
+ * Default {@link NetworkProbe}: an active `dns.lookup` against `host`.
+ *
+ * Deliberately does not hit a hardcoded external address (e.g. `1.1.1.1`) —
+ * probing a host that has already answered successfully during this crawl
+ * session (see `choose-probe-host.ts`) avoids depending on infrastructure
+ * outside the sites actually being crawled. `dns.lookup` alone is enough:
+ * it exercises the operator's local resolver / network path, which is
+ * exactly the layer an operator-side outage breaks.
+ * @param host - Hostname to resolve.
+ * @returns `true` if the lookup succeeds, `false` on any error (including
+ *   NXDOMAIN, timeout, or resolver unavailability).
+ */
+export declare function probeNetwork(host: string): Promise<boolean>;

package/lib/crawler/probe-network.js

@@ -0,0 +1,21 @@
+import { lookup } from 'node:dns';
+/**
+ * Default {@link NetworkProbe}: an active `dns.lookup` against `host`.
+ *
+ * Deliberately does not hit a hardcoded external address (e.g. `1.1.1.1`) —
+ * probing a host that has already answered successfully during this crawl
+ * session (see `choose-probe-host.ts`) avoids depending on infrastructure
+ * outside the sites actually being crawled. `dns.lookup` alone is enough:
+ * it exercises the operator's local resolver / network path, which is
+ * exactly the layer an operator-side outage breaks.
+ * @param host - Hostname to resolve.
+ * @returns `true` if the lookup succeeds, `false` on any error (including
+ *   NXDOMAIN, timeout, or resolver unavailability).
+ */
+export function probeNetwork(host) {
+    return new Promise((resolve) => {
+        lookup(host, (error) => {
+            resolve(!error);
+        });
+    });
+}

package/lib/crawler/types.d.ts

@@ -1,6 +1,8 @@
+import type { NetworkProbe } from './probe-network.js';
 import type { PageSource } from '../archive/types.js';
+import type { ErrorKind } from '../types.js';
 import type { PageData, CrawlerError, Resource } from '../utils/types/types.js';
-import type { ChangePhaseEvent, ScrapeResult } from '@d-zero/beholder';
+import type { ChangePhaseEvent, ConsoleLogEntry, ScrapeResult } from '@d-zero/beholder';
 import type { ParseURLOptions } from '@d-zero/shared/parse-url';
 /**
  * Result of resolving a URL that redirects to a destination already rendered
@@ -160,6 +162,34 @@ export interface CrawlerOptions extends Required<Pick<ParseURLOptions, 'disableQ
      * the DB DEFAULT `'crawled'` applies.
      */
     inventoryMode: InventoryMode | null;
+    /**
+     * Sliding-window size in ms for network-outage suspicion. See
+     * `NetworkOutageDetector`.
+     */
+    networkOutageWindowMs: number;
+    /**
+     * Minimum error count within {@link networkOutageWindowMs} to declare a
+     * suspect outage.
+     */
+    networkOutageErrorThreshold: number;
+    /**
+     * Minimum distinct-host count within {@link networkOutageWindowMs} to
+     * declare a suspect outage. Guards against a single flaky host looking
+     * like a network-wide event.
+     */
+    networkOutageHostThreshold: number;
+    /**
+     * Interval in ms between recovery probes while the network gate is
+     * closed.
+     */
+    networkOutageProbeIntervalMs: number;
+    /**
+     * Injectable network-reachability probe, or `null` to use the default
+     * `dns.lookup`-based `probeNetwork`. Overriding this is the seam tests
+     * use to simulate confirmed outages and recoveries deterministically
+     * without touching the real network.
+     */
+    networkProbe: NetworkProbe | null;
 }
 /**
  * Inventory-mode runtime configuration. Passed from
@@ -294,6 +324,31 @@ export interface CrawlerEventTypes {
         /** The URL of the referenced resource (without hash). */
         src: string;
     };
+    /**
+     * Emitted once per scrape with the console messages / page errors
+     * beholder captured for that page (issue #228). Only emitted when
+     * `entries` is non-empty — see `Crawler#handleConsoleLogs` for why a
+     * degraded re-scrape that captures nothing must not clear prior good
+     * data.
+     */
+    consoleLogs: {
+        /**
+         * The originally-requested URL, normalised (`withoutHashAndAuth`
+         * form) — the same identity `updatePage` resolves its redirect
+         * chain from, NOT necessarily the page that ends up holding the
+         * content.
+         */
+        pageUrl: string;
+        /**
+         * The redirect chain hops captured during fetch, in order. Empty
+         * when the page was not redirected, or when the scrape produced no
+         * `pageData` (a `'skipped'` / `'error'` result) and no redirect
+         * information is available.
+         */
+        redirectPaths: readonly string[];
+        /** The captured console messages / page errors, in capture order. */
+        entries: ConsoleLogEntry[];
+    };
     /**
      * Emitted when the entire crawl process has completed or been aborted.
      */
@@ -349,4 +404,95 @@ export interface CrawlerEventTypes {
          */
         source: PageSource | undefined;
     };
+    /**
+     * Emitted the instant `Crawler` closes its internal network gate after a
+     * recovery probe CONFIRMS a suspect outage (the sliding-window threshold
+     * alone only makes it a suspect — see `NetworkOutageDetector`). The
+     * orchestrator persists this via `Archive.insertNetworkOutage` and must
+     * remember the returned row id to pass to the matching
+     * `networkOutageRecovered` event, since `Crawler` itself never touches
+     * the archive and has no way to know the row's id.
+     */
+    networkOutageConfirmed: {
+        /** Backdated to the earliest error still inside the detector's window at trigger time. */
+        startedAt: number;
+        /** When the sliding window actually crossed both thresholds. */
+        detectedAt: number;
+        /** Hostname the recovery probe is targeting, or `null` if none was available. */
+        probeHost: string | null;
+        triggerErrorCount: number;
+        triggerHostCount: number;
+    };
+    /**
+     * Emitted the instant `Crawler` reopens its internal network gate after
+     * a recovery probe succeeds. NOT emitted when the gate is opened
+     * because the crawl was aborted while paused — in that case the outage
+     * row is deliberately left open for the next writer session's
+     * boot-time finalizer to resolve (see
+     * `db-ops/outages/close-stale-open-network-outages.ts`), since an abort
+     * says nothing about whether the network actually recovered.
+     */
+    networkOutageRecovered: {
+        /** Epoch ms the recovery probe first succeeded. */
+        endedAt: number;
+    };
+}
+/**
+ * Tunables for `NetworkOutageDetector`.
+ */
+export interface NetworkOutageDetectorOptions {
+    /**
+     * Sliding-window size in ms (`W`). Before each check, entries older than
+     * `at - windowMs` (inclusive boundary — an entry exactly `windowMs` old
+     * still counts) are evicted.
+     */
+    readonly windowMs: number;
+    /** Minimum error count within the window to declare a suspect outage (`N`). */
+    readonly errorThreshold: number;
+    /**
+     * Minimum number of DISTINCT hosts represented in the window to declare
+     * a suspect outage (`M`). Guards against one flaky host (a site that is
+     * genuinely retrying/failing on its own) looking like a network-wide
+     * event — a real local-network blip surfaces across unrelated hosts at
+     * once.
+     */
+    readonly hostThreshold: number;
+}
+/**
+ * One observed error, as fed to `NetworkOutageDetector.record`.
+ */
+export interface NetworkErrorRecord {
+    readonly kind: ErrorKind;
+    readonly host: string;
+    /**
+     * Epoch ms this error was observed. Caller-supplied — the detector never
+     * calls `Date.now()` itself, so window-boundary behaviour can be pinned
+     * with exact values instead of fake timers. Callers MUST supply
+     * non-decreasing values across successive `record()` calls; the window
+     * eviction is a simple cutoff against the latest `at` and does not
+     * re-sort out-of-order input.
+     */
+    readonly at: number;
+}
+/**
+ * Emitted by `NetworkOutageDetector.record` the instant the sliding window
+ * crosses both thresholds.
+ */
+export interface OutageSuspect {
+    /**
+     * Backdated to the earliest error still inside the window at trigger
+     * time — NOT the trigger instant itself. A sliding-window detector only
+     * confirms an outage after `W` seconds and `N` errors have accumulated,
+     * so the outage itself started earlier; backdating lets the persisted
+     * `network_outages` row (and the failures it retroactively covers) reach
+     * back to that earlier point instead of losing everything the detector
+     * missed while still accumulating evidence.
+     */
+    readonly startedAt: number;
+    /** The `at` of the record that tripped the threshold. */
+    readonly detectedAt: number;
+    /** Window size at trigger time (== `errorThreshold` or more). */
+    readonly triggerErrorCount: number;
+    /** Distinct host count at trigger time (== `hostThreshold` or more). */
+    readonly triggerHostCount: number;
 }