@nitpicker/crawler 0.14.0 → 0.16.0

package/lib/crawler/dedupe/types.d.ts

@@ -0,0 +1,42 @@
+/** Options controlling the opt-in same-cluster soft cap (`--dedupe-cap` / `--dedupe-map-cap`). */
+export interface DedupeCapOptions {
+    /** Base Misra-Gries majority-vote threshold, before confidence-signal halving. */
+    cap: number;
+    /** Hard cap on the number of distinct shapes tracked at once; the least-recently-touched shape is evicted beyond this. */
+    mapCap: number;
+}
+/** Per-shape Misra-Gries majority-vote slot. */
+export interface DedupeSlot {
+    /** The dominant meta signature currently "winning" for this shape. */
+    metaSig: string;
+    /** Majority-vote counter: incremented on a matching `metaSig`, decremented otherwise. Never exceeds the true count of matching observations. */
+    count: number;
+    /** The `computeBodyHash` result recorded when this slot was (re)created. */
+    bodyHash: Buffer;
+}
+/**
+ * One page's observation fed into `DedupeCapTracker#observe`. Callers are
+ * responsible for excluding pages with no signal (empty `computeMetaSignature`
+ * result, external, or metadata-only) before constructing this — the tracker
+ * itself does not special-case them.
+ */
+export interface DedupeCapObservation {
+    /** The page's URL shape key (see `computeShapeKey`). */
+    shapeKey: string;
+    /** The page's meta signature (see `computeMetaSignature`). */
+    metaSig: string;
+    /** The page's `computeBodyHash` result. */
+    bodyHash: Buffer;
+    /** Whether the page's (absolutised) `og:url` differs from its own URL (see `resolveOgUrlMismatch`). */
+    ogUrlMismatch: boolean;
+    /** The page's own URL — recorded as the resulting event's `sampleUrl` if this observation caps the shape. */
+    url: string;
+}
+/** Emitted the moment a shape's effective threshold is first reached. */
+export interface DedupeCapEvent {
+    shapeKey: string;
+    sampleUrl: string;
+    bodyHash: Buffer;
+    effectiveThreshold: number;
+    observedCount: number;
+}

package/lib/crawler/dedupe/types.js

@@ -0,0 +1 @@
+export {};

package/lib/crawler/dns-burned-host-burn-timestamps.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Companion to `dnsBurnedHostCache`: records the epoch-ms timestamp of each
+ * host burned by THIS session's `#sendHeadRequest` `onGiveUp` path (the
+ * `shouldBurnHost`-gated write in `crawler.ts`).
+ *
+ * Deliberately NOT populated by `#preloadDnsBurnedHostCache`'s seeding from
+ * a previous session's `crawl_errors` — those entries represent hosts
+ * already proven dead across sessions and must never be evicted just
+ * because THIS session's network happened to recover from an outage. Since
+ * `dnsBurnedHostCache`'s value (`ErrorKind`) carries no provenance of its
+ * own (preload-seeded and session-learned burns are indistinguishable by
+ * value alone), this separate timestamp map is what lets
+ * `evict-outage-tainted-dns-burns.ts` tell them apart: only hosts present
+ * HERE are eligible for eviction, and only if their burn timestamp falls
+ * inside the just-recovered outage's window.
+ */
+export declare const dnsBurnedHostBurnTimestamps: Map<string, number>;

package/lib/crawler/dns-burned-host-burn-timestamps.js

@@ -0,0 +1,17 @@
+/**
+ * Companion to `dnsBurnedHostCache`: records the epoch-ms timestamp of each
+ * host burned by THIS session's `#sendHeadRequest` `onGiveUp` path (the
+ * `shouldBurnHost`-gated write in `crawler.ts`).
+ *
+ * Deliberately NOT populated by `#preloadDnsBurnedHostCache`'s seeding from
+ * a previous session's `crawl_errors` — those entries represent hosts
+ * already proven dead across sessions and must never be evicted just
+ * because THIS session's network happened to recover from an outage. Since
+ * `dnsBurnedHostCache`'s value (`ErrorKind`) carries no provenance of its
+ * own (preload-seeded and session-learned burns are indistinguishable by
+ * value alone), this separate timestamp map is what lets
+ * `evict-outage-tainted-dns-burns.ts` tell them apart: only hosts present
+ * HERE are eligible for eviction, and only if their burn timestamp falls
+ * inside the just-recovered outage's window.
+ */
+export const dnsBurnedHostBurnTimestamps = new Map();

package/lib/crawler/evict-network-classified-destination-cache-entries.d.ts

@@ -0,0 +1,26 @@
+import type { PageData } from '@d-zero/beholder';
+/**
+ * Delete every `destinationCache` entry whose cached value is an `Error`
+ * that classifies as a {@link NETWORK_RELATED_ERROR_KINDS network-related}
+ * kind, leaving successes and site-specific failures (`tls`,
+ * `client-blocked`, `connection-refused`, …) untouched.
+ *
+ * Called by `Crawler` whenever its network gate transitions from closed to
+ * open (whether via a successful recovery probe or an abort), so a URL
+ * that failed with `getaddrinfo ENOTFOUND` only because the operator's own
+ * network was down does not stay wrongly cached as "this host is dead" for
+ * the rest of the session.
+ *
+ * Deliberately does NOT distinguish "cached during THIS specific outage"
+ * from "cached during an earlier blip this session" — any cached error
+ * whose KIND looks network-related is, by definition, potentially stale
+ * evidence about the operator's network rather than the target site, so it
+ * is always safe to re-test after any recovery. The trade-off is a few
+ * redundant HEAD attempts for hosts that were already dead independent of
+ * the outage; the alternative (a stale "outage-tainted" failure verdict
+ * surviving the rest of the session) is strictly worse.
+ * @param cache - The cache to sweep. Takes the `Map` explicitly (rather
+ *   than importing the `destinationCache` singleton directly) so this stays
+ *   unit-testable without touching module-level state.
+ */
+export declare function evictNetworkClassifiedDestinationCacheEntries(cache: Map<string, PageData | Error>): void;

package/lib/crawler/evict-network-classified-destination-cache-entries.js

@@ -0,0 +1,34 @@
+import { classifyErrorKind } from '../classify-error-kind.js';
+import { NETWORK_RELATED_ERROR_KINDS } from '../network-related-error-kinds.js';
+/**
+ * Delete every `destinationCache` entry whose cached value is an `Error`
+ * that classifies as a {@link NETWORK_RELATED_ERROR_KINDS network-related}
+ * kind, leaving successes and site-specific failures (`tls`,
+ * `client-blocked`, `connection-refused`, …) untouched.
+ *
+ * Called by `Crawler` whenever its network gate transitions from closed to
+ * open (whether via a successful recovery probe or an abort), so a URL
+ * that failed with `getaddrinfo ENOTFOUND` only because the operator's own
+ * network was down does not stay wrongly cached as "this host is dead" for
+ * the rest of the session.
+ *
+ * Deliberately does NOT distinguish "cached during THIS specific outage"
+ * from "cached during an earlier blip this session" — any cached error
+ * whose KIND looks network-related is, by definition, potentially stale
+ * evidence about the operator's network rather than the target site, so it
+ * is always safe to re-test after any recovery. The trade-off is a few
+ * redundant HEAD attempts for hosts that were already dead independent of
+ * the outage; the alternative (a stale "outage-tainted" failure verdict
+ * surviving the rest of the session) is strictly worse.
+ * @param cache - The cache to sweep. Takes the `Map` explicitly (rather
+ *   than importing the `destinationCache` singleton directly) so this stays
+ *   unit-testable without touching module-level state.
+ */
+export function evictNetworkClassifiedDestinationCacheEntries(cache) {
+    for (const [key, value] of cache) {
+        if (value instanceof Error &&
+            NETWORK_RELATED_ERROR_KINDS.has(classifyErrorKind(value.message))) {
+            cache.delete(key);
+        }
+    }
+}

package/lib/crawler/evict-outage-tainted-dns-burns.d.ts

@@ -0,0 +1,23 @@
+import type { OutageWindow } from '../is-within-outage-window.js';
+import type { ErrorKind } from '../types.js';
+/**
+ * Undo `dnsBurnedHostCache` burns whose recorded timestamp (see
+ * `dns-burned-host-burn-timestamps.ts`) falls inside `window` — i.e. hosts
+ * THIS session burned because a HEAD request failed with a
+ * `dns`-classified error while (or just before) the operator's own network
+ * was down, not because the host is actually dead.
+ *
+ * Preload-seeded burns are structurally immune: they never appear in
+ * `burnTimestamps` (see that module's docstring), so this function can
+ * never touch them regardless of the window.
+ * @param options - Named parameters.
+ * @param options.cache - `dnsBurnedHostCache` (or a test double with the same shape).
+ * @param options.burnTimestamps - `dnsBurnedHostBurnTimestamps` (or a test double).
+ * @param options.window - The just-resolved outage window (`{ startedAt, endedAt }`)
+ *   to test each burn's timestamp against.
+ */
+export declare function evictOutageTaintedDnsBurns(options: {
+    cache: Map<string, ErrorKind>;
+    burnTimestamps: Map<string, number>;
+    window: OutageWindow;
+}): void;

package/lib/crawler/evict-outage-tainted-dns-burns.js

@@ -0,0 +1,26 @@
+import { isWithinOutageWindow } from '../is-within-outage-window.js';
+/**
+ * Undo `dnsBurnedHostCache` burns whose recorded timestamp (see
+ * `dns-burned-host-burn-timestamps.ts`) falls inside `window` — i.e. hosts
+ * THIS session burned because a HEAD request failed with a
+ * `dns`-classified error while (or just before) the operator's own network
+ * was down, not because the host is actually dead.
+ *
+ * Preload-seeded burns are structurally immune: they never appear in
+ * `burnTimestamps` (see that module's docstring), so this function can
+ * never touch them regardless of the window.
+ * @param options - Named parameters.
+ * @param options.cache - `dnsBurnedHostCache` (or a test double with the same shape).
+ * @param options.burnTimestamps - `dnsBurnedHostBurnTimestamps` (or a test double).
+ * @param options.window - The just-resolved outage window (`{ startedAt, endedAt }`)
+ *   to test each burn's timestamp against.
+ */
+export function evictOutageTaintedDnsBurns(options) {
+    const { cache, burnTimestamps, window } = options;
+    for (const [host, burnedAt] of burnTimestamps) {
+        if (isWithinOutageWindow(burnedAt, [window])) {
+            cache.delete(host);
+            burnTimestamps.delete(host);
+        }
+    }
+}

package/lib/crawler/fetch-destination.js

@@ -1,5 +1,6 @@
 import { delay } from '@d-zero/shared/delay';
 import redirects from 'follow-redirects';
+import { decodeAuthCredential } from './decode-auth-credential.js';
 import { destinationCache } from './destination-cache.js';
 import NetTimeoutError from './net-timeout-error.js';
 import { shouldGetFallbackOnHeadFailure } from './should-get-fallback-on-head-failure.js';
@@ -140,8 +141,19 @@ async function _fetchHead(url, isExternal, method, titleBytesLimit, userAgent, t
                 // Range: url.extname?.toLowerCase() === 'pdf' ? 'bytes=0-0' : undefined,
             },
         };
-        if (url.username && url.password) {
-            request.auth = `${url.username}:${url.password}`;
+        // `||`, not `&&`: ExURL rounds an empty userinfo component to null,
+        // so `http://user:@host/` (legal empty-password Basic auth) has
+        // `password: null`. Requiring both would leave this pre-flight
+        // unauthenticated while the browser path (`page.authenticate`, which
+        // always runs and treats null as '') succeeds — and for non-HTML
+        // content the browser never runs, so the HEAD 401 would become the
+        // page's final recorded status.
+        if (url.username || url.password) {
+            // ExURL fields are WHATWG percent-encoded; `auth` is base64'd
+            // verbatim into the Authorization header, so decode first —
+            // mirroring what Node's own `urlToOptions` does for
+            // `http.request(url)` (see `decode-auth-credential.ts`).
+            request.auth = `${decodeAuthCredential(url.username)}:${decodeAuthCredential(url.password)}`;
         }
         let req;
         let destroyed = false;

package/lib/crawler/generate-predicted-urls.d.ts

@@ -5,6 +5,18 @@ import type { PaginationPattern } from './types.js';
  * Starting from `currentUrl`, applies the pattern's step `count` times to produce
  * future page URLs (e.g. if step=1 and currentNumber=2, generates page 3, 4, ...).
  * These URLs are pushed into the crawl queue and discarded later if they 404.
+ *
+ * Generation stops at the first token that would lose its original digit-string
+ * shape, rather than skipping it and continuing: beyond `Number.MAX_SAFE_INTEGER`,
+ * `String()` renders scientific notation (e.g. `"1e+21"`), which is not a valid
+ * path/query token and — left unguarded — becomes a self-generated URL that no
+ * page on the target site ever linked to (observed in production: a
+ * `/news/date/{year}/` pager whose per-anchor `step` was miscalculated from
+ * unrelated pages, compounding across rounds until it emitted
+ * `1.7715854126052197e+120`). A token growing far beyond its original digit
+ * count is equally implausible as a next page number. Both trends are
+ * monotonic as `i` increases, so once one prediction is rejected, every later
+ * one in the same batch would be too — there is nothing to skip past.
  * @param pattern - The detected pagination pattern from `detectPaginationPattern()`
  * @param currentUrl - The URL to extrapolate from (protocol-agnostic, without hash/auth)
  * @param count - Number of predicted URLs to generate (typically equals concurrency)

package/lib/crawler/generate-predicted-urls.js

@@ -1,11 +1,24 @@
 import { decomposeUrl } from './decompose-url.js';
 import { reconstructUrl } from './reconstruct-url.js';
+const DIGITS_ONLY_PATTERN = /^\d+$/;
 /**
  * Generates predicted URLs by extrapolating the detected pagination pattern.
  *
  * Starting from `currentUrl`, applies the pattern's step `count` times to produce
  * future page URLs (e.g. if step=1 and currentNumber=2, generates page 3, 4, ...).
  * These URLs are pushed into the crawl queue and discarded later if they 404.
+ *
+ * Generation stops at the first token that would lose its original digit-string
+ * shape, rather than skipping it and continuing: beyond `Number.MAX_SAFE_INTEGER`,
+ * `String()` renders scientific notation (e.g. `"1e+21"`), which is not a valid
+ * path/query token and — left unguarded — becomes a self-generated URL that no
+ * page on the target site ever linked to (observed in production: a
+ * `/news/date/{year}/` pager whose per-anchor `step` was miscalculated from
+ * unrelated pages, compounding across rounds until it emitted
+ * `1.7715854126052197e+120`). A token growing far beyond its original digit
+ * count is equally implausible as a next page number. Both trends are
+ * monotonic as `i` increases, so once one prediction is rejected, every later
+ * one in the same batch would be too — there is nothing to skip past.
  * @param pattern - The detected pagination pattern from `detectPaginationPattern()`
  * @param currentUrl - The URL to extrapolate from (protocol-agnostic, without hash/auth)
  * @param count - Number of predicted URLs to generate (typically equals concurrency)
@@ -17,11 +30,29 @@ export function generatePredictedUrls(pattern, currentUrl, count) {
     const decomposed = decomposeUrl(currentUrl);
     if (!decomposed)
         return [];
+    const { pathSegments, queryValues } = decomposed;
+    const originalToken = pattern.tokenIndex < pathSegments.length
+        ? pathSegments[pattern.tokenIndex]
+        : queryValues[pattern.tokenIndex - pathSegments.length];
+    if (originalToken === undefined)
+        return [];
+    // A pager jumping from 4 digits to 5 (e.g. year 9999 → 10000) is plausible;
+    // jumping straight to 6+ digits within the same predicted batch is not.
+    const maxDigits = originalToken.length + 1;
     const results = [];
     for (let i = 1; i <= count; i++) {
         const nextNum = pattern.currentNumber + pattern.step * i;
-        const url = reconstructUrl(decomposed, pattern.tokenIndex, String(nextNum));
-        results.push(url);
+        if (!Number.isSafeInteger(nextNum))
+            break;
+        const rendered = String(nextNum);
+        if (!DIGITS_ONLY_PATTERN.test(rendered))
+            break;
+        // Preserve zero-padding width (e.g. "01" → "02"), without truncating a
+        // value that has legitimately grown past the original width.
+        const padded = rendered.padStart(originalToken.length, '0');
+        if (padded.length > maxDigits)
+            break;
+        results.push(reconstructUrl(decomposed, pattern.tokenIndex, padded));
     }
     return results;
 }

package/lib/crawler/is-puppeteer-fallback-candidate.js

@@ -26,6 +26,9 @@ import { classifyErrorKind } from '../classify-error-kind.js';
  *   as `timeout` via the `NetTimeoutError "Timeout: <url>"` shape.
  * - **local-network** — operator-side connectivity loss; nothing on this
  *   machine will reach the host.
+ * - **redirect-loop** — the redirect chain itself never terminates; a
+ *   puppeteer navigation follows the identical Location-header chain (or
+ *   hits Chromium's own `ERR_TOO_MANY_REDIRECTS`) and loops the same way.
  * - **protocol** — puppeteer lifecycle race; bouncing back to puppeteer
  *   reproduces the same race.
  * - **unknown** — by design. Spinning up a fresh Chromium for every

package/lib/crawler/network-gate.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Level-triggered gate that dealer worker callbacks await before doing any
+ * network work. Achieves "pause the crawl until the network recovers"
+ * without touching `@d-zero/dealer` internals — a closed gate just makes
+ * the worker's task body a long-running `await`, which is all dealer ever
+ * sees.
+ *
+ * **`epoch`** counts outage generations: it increments on every `close()`
+ * transition (open → closed) and holds that value for the remainder of the
+ * closed period AND through the following open period, until the next
+ * `close()`. This is the hook `destinationCache` / `dnsBurnedHostCache`
+ * eviction uses: an entry written while the gate was closed is tagged with
+ * `gate.epoch` at write time; after `open()`, the caller purges entries
+ * tagged with that same epoch value. Because `epoch` only advances on
+ * `close()` (never on `open()`), entries written during ordinary (open)
+ * operation are never mistakenly tagged with a soon-to-be-purged epoch —
+ * only writes that happen while `isOpen` is `false` get tagged at all.
+ */
+export default class NetworkGate {
+    #private;
+    /** `true` when the gate is open (the default). */
+    get isOpen(): boolean;
+    /**
+     * Current outage-generation counter. See the class docstring for how
+     * cache-eviction callers are expected to use this value.
+     */
+    get epoch(): number;
+    /**
+     * Close the gate, blocking future {@link wait} callers until the next
+     * {@link open}. Idempotent — calling `close()` while already closed does
+     * nothing (in particular, it does NOT bump {@link epoch} a second time,
+     * which would otherwise let a still-open outage's cache entries escape
+     * eviction under a stale epoch number).
+     */
+    close(): void;
+    /**
+     * Open the gate, resolving every {@link wait} caller currently pending —
+     * including ones that subscribed after `close()` but before this call.
+     * Idempotent — calling `open()` while already open does nothing.
+     */
+    open(): void;
+    /**
+     * Resolve immediately if the gate is open; otherwise resolve on the next
+     * {@link open}. Multiple concurrent callers all resolve on the same
+     * `open()` call — no awaiter is dropped regardless of when it subscribed
+     * relative to others.
+     */
+    wait(): Promise<void>;
+}

package/lib/crawler/network-gate.js

@@ -0,0 +1,78 @@
+/**
+ * Level-triggered gate that dealer worker callbacks await before doing any
+ * network work. Achieves "pause the crawl until the network recovers"
+ * without touching `@d-zero/dealer` internals — a closed gate just makes
+ * the worker's task body a long-running `await`, which is all dealer ever
+ * sees.
+ *
+ * **`epoch`** counts outage generations: it increments on every `close()`
+ * transition (open → closed) and holds that value for the remainder of the
+ * closed period AND through the following open period, until the next
+ * `close()`. This is the hook `destinationCache` / `dnsBurnedHostCache`
+ * eviction uses: an entry written while the gate was closed is tagged with
+ * `gate.epoch` at write time; after `open()`, the caller purges entries
+ * tagged with that same epoch value. Because `epoch` only advances on
+ * `close()` (never on `open()`), entries written during ordinary (open)
+ * operation are never mistakenly tagged with a soon-to-be-purged epoch —
+ * only writes that happen while `isOpen` is `false` get tagged at all.
+ */
+export default class NetworkGate {
+    #epoch = 0;
+    #open = true;
+    #waiters = [];
+    /** `true` when the gate is open (the default). */
+    get isOpen() {
+        return this.#open;
+    }
+    /**
+     * Current outage-generation counter. See the class docstring for how
+     * cache-eviction callers are expected to use this value.
+     */
+    get epoch() {
+        return this.#epoch;
+    }
+    /**
+     * Close the gate, blocking future {@link wait} callers until the next
+     * {@link open}. Idempotent — calling `close()` while already closed does
+     * nothing (in particular, it does NOT bump {@link epoch} a second time,
+     * which would otherwise let a still-open outage's cache entries escape
+     * eviction under a stale epoch number).
+     */
+    close() {
+        if (!this.#open) {
+            return;
+        }
+        this.#open = false;
+        this.#epoch += 1;
+    }
+    /**
+     * Open the gate, resolving every {@link wait} caller currently pending —
+     * including ones that subscribed after `close()` but before this call.
+     * Idempotent — calling `open()` while already open does nothing.
+     */
+    open() {
+        if (this.#open) {
+            return;
+        }
+        this.#open = true;
+        const waiters = this.#waiters;
+        this.#waiters = [];
+        for (const resolve of waiters) {
+            resolve();
+        }
+    }
+    /**
+     * Resolve immediately if the gate is open; otherwise resolve on the next
+     * {@link open}. Multiple concurrent callers all resolve on the same
+     * `open()` call — no awaiter is dropped regardless of when it subscribed
+     * relative to others.
+     */
+    wait() {
+        if (this.#open) {
+            return Promise.resolve();
+        }
+        return new Promise((resolve) => {
+            this.#waiters.push(resolve);
+        });
+    }
+}

package/lib/crawler/network-outage-detector.d.ts

@@ -0,0 +1,51 @@
+import type { NetworkErrorRecord, NetworkOutageDetectorOptions, OutageSuspect } from './types.js';
+/**
+ * Sliding-window detector for "this looks like the crawl operator's own
+ * network, not the target sites, is having a bad time" — the trigger that
+ * hands off to an active probe (see `probe-network.ts`) before the crawler
+ * commits to pausing.
+ *
+ * Deliberately dependency-free: no timers, no DB, no `Crawler` instance.
+ * Time flows in via {@link NetworkErrorRecord.at} only, which is what makes
+ * window-boundary behaviour testable with exact values instead of
+ * `vi.useFakeTimers()`.
+ * @example
+ * ```ts
+ * const detector = new NetworkOutageDetector({
+ *   windowMs: 10_000,
+ *   errorThreshold: 5,
+ *   hostThreshold: 2,
+ * });
+ * const suspect = detector.record({ kind: 'dns', host: 'a.example', at: Date.now() });
+ * if (suspect) {
+ *   // probe before closing the gate
+ * }
+ * ```
+ */
+export default class NetworkOutageDetector {
+    #private;
+    constructor(options: NetworkOutageDetectorOptions);
+    /**
+     * Record one observed error and check whether it tips the sliding window
+     * over both thresholds.
+     *
+     * Non-network `kind`s (see `NETWORK_RELATED_ERROR_KINDS`) are silently
+     * ignored: they never enter the window and can never contribute to a
+     * trigger, regardless of how many arrive.
+     * @param record - The observed error.
+     * @param record.kind
+     * @param record.host
+     * @param record.at
+     * @returns An {@link OutageSuspect} the instant both thresholds are met,
+     *   or `null` otherwise. On a trigger, the window is cleared so the same
+     *   batch of errors cannot fire a second time on the next call.
+     */
+    record({ kind, host, at }: NetworkErrorRecord): OutageSuspect | null;
+    /**
+     * Clear the sliding window. Called at the start of a fresh crawl session
+     * (`Crawler.#runDeal`) so error observations from a prior session on the
+     * same `Crawler` instance never carry over — mirrors `#successfulHosts
+     * .clear()` / `#scrapedDestinations.clear()`'s per-session reset.
+     */
+    reset(): void;
+}

package/lib/crawler/network-outage-detector.js

@@ -0,0 +1,81 @@
+import { NETWORK_RELATED_ERROR_KINDS } from '../network-related-error-kinds.js';
+/**
+ * Sliding-window detector for "this looks like the crawl operator's own
+ * network, not the target sites, is having a bad time" — the trigger that
+ * hands off to an active probe (see `probe-network.ts`) before the crawler
+ * commits to pausing.
+ *
+ * Deliberately dependency-free: no timers, no DB, no `Crawler` instance.
+ * Time flows in via {@link NetworkErrorRecord.at} only, which is what makes
+ * window-boundary behaviour testable with exact values instead of
+ * `vi.useFakeTimers()`.
+ * @example
+ * ```ts
+ * const detector = new NetworkOutageDetector({
+ *   windowMs: 10_000,
+ *   errorThreshold: 5,
+ *   hostThreshold: 2,
+ * });
+ * const suspect = detector.record({ kind: 'dns', host: 'a.example', at: Date.now() });
+ * if (suspect) {
+ *   // probe before closing the gate
+ * }
+ * ```
+ */
+export default class NetworkOutageDetector {
+    #entries = [];
+    #errorThreshold;
+    #hostThreshold;
+    #windowMs;
+    constructor(options) {
+        this.#windowMs = options.windowMs;
+        this.#errorThreshold = options.errorThreshold;
+        this.#hostThreshold = options.hostThreshold;
+    }
+    /**
+     * Record one observed error and check whether it tips the sliding window
+     * over both thresholds.
+     *
+     * Non-network `kind`s (see `NETWORK_RELATED_ERROR_KINDS`) are silently
+     * ignored: they never enter the window and can never contribute to a
+     * trigger, regardless of how many arrive.
+     * @param record - The observed error.
+     * @param record.kind
+     * @param record.host
+     * @param record.at
+     * @returns An {@link OutageSuspect} the instant both thresholds are met,
+     *   or `null` otherwise. On a trigger, the window is cleared so the same
+     *   batch of errors cannot fire a second time on the next call.
+     */
+    record({ kind, host, at }) {
+        if (!NETWORK_RELATED_ERROR_KINDS.has(kind)) {
+            return null;
+        }
+        const cutoff = at - this.#windowMs;
+        this.#entries = this.#entries.filter((entry) => entry.at >= cutoff);
+        this.#entries.push({ host, at });
+        const distinctHosts = new Set(this.#entries.map((entry) => entry.host)).size;
+        if (this.#entries.length < this.#errorThreshold ||
+            distinctHosts < this.#hostThreshold) {
+            return null;
+        }
+        const startedAt = Math.min(...this.#entries.map((entry) => entry.at));
+        const suspect = {
+            startedAt,
+            detectedAt: at,
+            triggerErrorCount: this.#entries.length,
+            triggerHostCount: distinctHosts,
+        };
+        this.#entries = [];
+        return suspect;
+    }
+    /**
+     * Clear the sliding window. Called at the start of a fresh crawl session
+     * (`Crawler.#runDeal`) so error observations from a prior session on the
+     * same `Crawler` instance never carry over — mirrors `#successfulHosts
+     * .clear()` / `#scrapedDestinations.clear()`'s per-session reset.
+     */
+    reset() {
+        this.#entries = [];
+    }
+}

package/lib/crawler/network-outage-summary-counter.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Session-scoped counter for how many operator-network outages were
+ * confirmed, and their total duration. The orchestrator accumulates into
+ * this at every `networkOutageRecovered` event, reads it at crawl shutdown
+ * to emit a `[network] N outage(s), Ms total` summary line, then zeroes it
+ * for the next session (mirrors `dnsBurnedHostShortCircuitCounter`'s
+ * lifecycle, including its known limitation: two `CrawlerOrchestrator`
+ * crawls running concurrently in the same process would accumulate into and
+ * zero the same shared counter, cross-contaminating each other's summary.
+ * The CLI only ever runs one crawl per process, so this has never been
+ * observed in practice; fixing it would mean threading per-session state
+ * through both counters together, out of scope here).
+ *
+ * Exposed as a mutable object (not a plain `let`) so the counter remains a
+ * single shared reference across crawler / orchestrator imports — `let`
+ * bindings cannot be mutated from another module.
+ */
+export declare const networkOutageSummaryCounter: {
+    confirmedCount: number;
+    totalDurationMs: number;
+};

package/lib/crawler/network-outage-summary-counter.js

@@ -0,0 +1,18 @@
+/**
+ * Session-scoped counter for how many operator-network outages were
+ * confirmed, and their total duration. The orchestrator accumulates into
+ * this at every `networkOutageRecovered` event, reads it at crawl shutdown
+ * to emit a `[network] N outage(s), Ms total` summary line, then zeroes it
+ * for the next session (mirrors `dnsBurnedHostShortCircuitCounter`'s
+ * lifecycle, including its known limitation: two `CrawlerOrchestrator`
+ * crawls running concurrently in the same process would accumulate into and
+ * zero the same shared counter, cross-contaminating each other's summary.
+ * The CLI only ever runs one crawl per process, so this has never been
+ * observed in practice; fixing it would mean threading per-session state
+ * through both counters together, out of scope here).
+ *
+ * Exposed as a mutable object (not a plain `let`) so the counter remains a
+ * single shared reference across crawler / orchestrator imports — `let`
+ * bindings cannot be mutated from another module.
+ */
+export const networkOutageSummaryCounter = { confirmedCount: 0, totalDurationMs: 0 };

package/lib/crawler/probe-network.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Function shape for an injectable network-reachability probe: given a
+ * hostname, resolve to `true` if the network path to it appears healthy,
+ * `false` otherwise. Never rejects — a probe failure is a normal outcome
+ * (that's the whole point of probing), not an exceptional one.
+ *
+ * Injected via `CrawlerOptions.networkProbe` so tests can simulate
+ * confirmed outages and recoveries deterministically (a function that fails
+ * N times then starts succeeding) without touching the real network — the
+ * same injection pattern as `ResourceLookup` / `PageSourceLookup`.
+ */
+export type NetworkProbe = (host: string) => Promise<boolean>;
+/**
+ * Default {@link NetworkProbe}: an active `dns.lookup` against `host`.
+ *
+ * Deliberately does not hit a hardcoded external address (e.g. `1.1.1.1`) —
+ * probing a host that has already answered successfully during this crawl
+ * session (see `choose-probe-host.ts`) avoids depending on infrastructure
+ * outside the sites actually being crawled. `dns.lookup` alone is enough:
+ * it exercises the operator's local resolver / network path, which is
+ * exactly the layer an operator-side outage breaks.
+ * @param host - Hostname to resolve.
+ * @returns `true` if the lookup succeeds, `false` on any error (including
+ *   NXDOMAIN, timeout, or resolver unavailability).
+ */
+export declare function probeNetwork(host: string): Promise<boolean>;