@nitpicker/crawler 0.13.0 → 0.15.0

package/lib/crawler/crawler.js

@@ -14,12 +14,17 @@ import { crawlerLog } from '../debug.js';
 import { buildJsRedirectEdge } from './build-js-redirect-edge.js';
 import { buildRedirectEvent } from './build-redirect-event.js';
 import { captureImageDomPaths } from './capture-image-dom-paths.js';
+import { chooseProbeHost } from './choose-probe-host.js';
 import { createChangePhaseHandler } from './create-change-phase-handler.js';
 import { derivePageSource } from './derive-page-source.js';
+import { destinationCache } from './destination-cache.js';
 import { detectPaginationPattern } from './detect-pagination-pattern.js';
+import { dnsBurnedHostBurnTimestamps } from './dns-burned-host-burn-timestamps.js';
 import { dnsBurnedHostCache } from './dns-burned-host-cache.js';
 import { dnsBurnedHostShortCircuitCounter } from './dns-burned-host-short-circuit-counter.js';
 import { drainPhaseErrors } from './drain-phase-errors.js';
+import { evictNetworkClassifiedDestinationCacheEntries } from './evict-network-classified-destination-cache-entries.js';
+import { evictOutageTaintedDnsBurns } from './evict-outage-tainted-dns-burns.js';
 import { fetchDestination } from './fetch-destination.js';
 import { findScopeEntry } from './find-scope-entry.js';
 import { formatCrawlProgress } from './format-crawl-progress.js';
@@ -35,9 +40,12 @@ import { isPuppeteerFallbackCandidate } from './is-puppeteer-fallback-candidate.
 import LinkList from './link-list.js';
 import { linkToPageData } from './link-to-page-data.js';
 import { logUndrainedPhaseErrors } from './log-undrained-phase-errors.js';
+import NetworkGate from './network-gate.js';
+import NetworkOutageDetector from './network-outage-detector.js';
 import { partitionUrlsByHtml } from './partition-urls-by-html.js';
 import { planSubResourceEmits } from './plan-sub-resource-emits.js';
 import { PreloadShortCircuitError } from './preload-short-circuit-error.js';
+import { probeNetwork } from './probe-network.js';
 import { protocolAgnosticKey } from './protocol-agnostic-key.js';
 import { redirectDestKey } from './redirect-dest-key.js';
 import { resourceToPageData } from './resource-to-page-data.js';
@@ -56,6 +64,14 @@ import { shouldSkipUrl } from './should-skip-url.js';
  * stay on the final (longest) timeout for any additional attempts.
  */
 const HEAD_TIMEOUT_ESCALATION_MS = [10_000, 30_000, 60_000];
+/** Default {@link CrawlerOptions.networkOutageWindowMs}. */
+const DEFAULT_NETWORK_OUTAGE_WINDOW_MS = 10_000;
+/** Default {@link CrawlerOptions.networkOutageErrorThreshold}. */
+const DEFAULT_NETWORK_OUTAGE_ERROR_THRESHOLD = 5;
+/** Default {@link CrawlerOptions.networkOutageHostThreshold}. */
+const DEFAULT_NETWORK_OUTAGE_HOST_THRESHOLD = 2;
+/** Default {@link CrawlerOptions.networkOutageProbeIntervalMs}. */
+const DEFAULT_NETWORK_OUTAGE_PROBE_INTERVAL_MS = 10_000;
 /**
  * The core crawler engine that discovers and scrapes web pages.
  *
@@ -72,8 +88,39 @@ class Crawler extends EventEmitter {
     #abortController = new AbortController();
     /** Tracks discovered URLs, their scrape status, and deduplication. */
     #linkList = new LinkList();
+    /**
+     * Gate every worker callback awaits before doing network work (see the
+     * worker body inside {@link #runDeal}). Open by default; closed by
+     * {@link #handleOutageSuspect} once a recovery probe confirms a suspect
+     * outage, reopened once a later probe succeeds. Re-opened defensively at
+     * the start of {@link #runDeal} (a no-op if already open) so a fresh
+     * session never inherits a closed gate from a prior anomalous one.
+     */
+    #networkGate = new NetworkGate();
+    /**
+     * Sliding-window detector for "the operator's own network, not the
+     * target sites, looks like it is down". Fed from {@link #sendHeadRequest}'s
+     * `onWait` / `onGiveUp`; a non-null {@link OutageSuspect} triggers
+     * {@link #handleOutageSuspect}. Reset at the start of {@link #runDeal}.
+     * Assigned in the constructor (not a field initializer) because it
+     * needs `this.#options`'s network-outage tunables.
+     */
+    #networkOutageDetector;
     /** Merged crawler configuration (user overrides + defaults). */
     #options;
+    /**
+     * Synchronous claim flag guarding the async gap between "a suspect
+     * outage arrived" and "the confirming probe settled" in
+     * {@link #handleOutageSuspect}. Without it, two workers whose HEAD
+     * requests both exhaust retries in quick succession could each start
+     * their own confirming probe while the gate is still open, and if both
+     * probes fail, both would close the gate and emit
+     * `networkOutageConfirmed` — creating two simultaneously-open
+     * `network_outages` rows for one ongoing outage. Checked and set
+     * synchronously (no `await` between the check and the set), which is
+     * race-free because JS has no thread-level interleaving.
+     */
+    #outageHandlingInProgress = false;
     /**
      * Phase errors observed during {@link Crawler._launchBrowserAndScrape},
      * buffered per URL href so they can be emitted as `pageError` events
@@ -149,10 +196,21 @@ class Crawler extends EventEmitter {
             verbose: options?.verbose ?? false,
             userAgent: options?.userAgent || `Nitpicker/${pkg.version}`,
             ignoreRobots: options?.ignoreRobots ?? false,
+            mainContentSelector: options?.mainContentSelector ?? null,
             lookupResource: options?.lookupResource ?? null,
             lookupPageSource: options?.lookupPageSource ?? null,
             inventoryMode: options?.inventoryMode ?? null,
+            networkOutageWindowMs: options?.networkOutageWindowMs ?? DEFAULT_NETWORK_OUTAGE_WINDOW_MS,
+            networkOutageErrorThreshold: options?.networkOutageErrorThreshold ?? DEFAULT_NETWORK_OUTAGE_ERROR_THRESHOLD,
+            networkOutageHostThreshold: options?.networkOutageHostThreshold ?? DEFAULT_NETWORK_OUTAGE_HOST_THRESHOLD,
+            networkOutageProbeIntervalMs: options?.networkOutageProbeIntervalMs ?? DEFAULT_NETWORK_OUTAGE_PROBE_INTERVAL_MS,
+            networkProbe: options?.networkProbe ?? null,
         };
+        this.#networkOutageDetector = new NetworkOutageDetector({
+            windowMs: this.#options.networkOutageWindowMs,
+            errorThreshold: this.#options.networkOutageErrorThreshold,
+            hostThreshold: this.#options.networkOutageHostThreshold,
+        });
         this.#robotsChecker = new RobotsChecker(this.#options.userAgent, !this.#options.ignoreRobots);
         for (const urlStr of this.#options.roots) {
             const url = parseUrl(urlStr, this.#options);
@@ -323,6 +381,84 @@ class Crawler extends EventEmitter {
             });
         }
     }
+    /**
+     * Emits captured console messages / page errors for a scrape (issue
+     * #228), skipping the emit entirely when `entries` is empty.
+     *
+     * The empty-skip is deliberate, not an optimization: `replaceConsoleLogs`
+     * replaces a page's rows wholesale (Scoped-Replace, like
+     * `anchor_edges` / `image_items`), so emitting on an empty capture would
+     * wipe out a prior good result on a degraded re-scrape (navigation
+     * timeout, partial render) that legitimately produced nothing this
+     * time — the same trade-off `updatePage` documents for anchors/images.
+     * @param entries - Console log entries captured during the page load.
+     * @param url - The originally-requested URL (not necessarily the page
+     *   that ends up holding the content — see `CrawlerEventTypes.consoleLogs`).
+     * @param redirectPaths - The redirect chain hops captured during fetch,
+     *   in order. Empty when the scrape produced no `pageData` (a
+     *   `'skipped'` / `'error'` result).
+     */
+    #handleConsoleLogs(entries, url, redirectPaths) {
+        if (entries.length === 0) {
+            return;
+        }
+        void this.emit('consoleLogs', {
+            pageUrl: url.withoutHashAndAuth,
+            redirectPaths,
+            entries,
+        });
+    }
+    /**
+     * Confirm a sliding-window suspect via an active probe, and if
+     * confirmed, close {@link #networkGate} and start
+     * {@link #runRecoveryProbeLoop}.
+     *
+     * Guarded by {@link #outageHandlingInProgress} (a synchronous
+     * check-then-set, race-free under JS's single-threaded execution) AND
+     * by `#networkGate.isOpen` — the latter covers the entire duration a
+     * recovery loop is running (no new suspect should re-confirm or
+     * re-probe while one outage is already open), the former covers only
+     * the narrow async gap between "decided to investigate" and "the
+     * confirming probe settled", which the gate-open check alone cannot see
+     * since the gate has not closed yet at that point.
+     * @param suspect - The trigger emitted by {@link NetworkOutageDetector.record}.
+     */
+    async #handleOutageSuspect(suspect) {
+        if (!this.#networkGate.isOpen || this.#outageHandlingInProgress) {
+            return;
+        }
+        this.#outageHandlingInProgress = true;
+        try {
+            // No usable probe target at all (no session successes yet AND no
+            // parseable root URL) — cannot confirm, and cannot ever detect
+            // recovery either, so there is nothing safe to do but leave the
+            // gate open and treat this as inconclusive.
+            const probeHost = chooseProbeHost(this.#successfulHosts, this.#options.roots);
+            if (probeHost === null) {
+                return;
+            }
+            const probe = this.#options.networkProbe ?? probeNetwork;
+            const initiallyReachable = await probe(probeHost);
+            if (initiallyReachable) {
+                // False alarm: the sliding window tripped (e.g. several
+                // unrelated hosts happened to fail close together) but the
+                // probe host answers fine. Leave the gate open.
+                return;
+            }
+            this.#networkGate.close();
+            void this.emit('networkOutageConfirmed', {
+                startedAt: suspect.startedAt,
+                detectedAt: suspect.detectedAt,
+                probeHost,
+                triggerErrorCount: suspect.triggerErrorCount,
+                triggerHostCount: suspect.triggerHostCount,
+            });
+            void this.#runRecoveryProbeLoop(probeHost, suspect.startedAt);
+        }
+        finally {
+            this.#outageHandlingInProgress = false;
+        }
+    }
     /**
      * Processes captured sub-resources from a page scrape, deduplicates them,
      * and emits `response` / `responseReferrers` events for new resources.
@@ -470,6 +606,54 @@ class Crawler extends EventEmitter {
             }
         }
     }
+    /**
+     * Undo cache damage from the outage window `[startedAt, endedAt]`:
+     * evict `destinationCache` entries whose cached error looks
+     * network-related (any such entry may be stale evidence about the
+     * operator's network, not the target site), and un-burn any
+     * `dnsBurnedHostCache` host THIS session burned during that window
+     * (preload-seeded burns are structurally immune — see
+     * `evict-outage-tainted-dns-burns.ts`).
+     *
+     * Called on every closed→open gate transition, whether triggered by a
+     * successful recovery probe or by an abort — the cached failures are
+     * stale either way, and the eviction itself has no failure mode that
+     * depends on why the gate reopened.
+     * @param startedAt - The outage's `startedAt` (from the triggering `OutageSuspect`).
+     * @param endedAt - The moment the gate is reopening.
+     */
+    #onGateReopened(startedAt, endedAt) {
+        evictNetworkClassifiedDestinationCacheEntries(destinationCache);
+        evictOutageTaintedDnsBurns({
+            cache: dnsBurnedHostCache,
+            burnTimestamps: dnsBurnedHostBurnTimestamps,
+            window: { startedAt, endedAt },
+        });
+    }
+    /**
+     * Feed one observed network-layer error into
+     * {@link #networkOutageDetector} and hand off to
+     * {@link #handleOutageSuspect} the instant its sliding window trips.
+     *
+     * Called from BOTH `onWait` (every non-final retry attempt) and
+     * `onGiveUp` (the final attempt) inside {@link #sendHeadRequest}, so a
+     * single URL's retry storm contributes every attempt's error, not just
+     * its terminal one — a real network-wide outage is expected to trip the
+     * `hostThreshold` gate from many DIFFERENT hosts' attempts arriving in
+     * the same short window, not from one URL retrying against one host.
+     * @param message - The raw error message to classify.
+     * @param host - Lower-cased hostname the error occurred on.
+     */
+    #recordNetworkError(message, host) {
+        const suspect = this.#networkOutageDetector.record({
+            kind: classifyErrorKind(message),
+            host,
+            at: Date.now(),
+        });
+        if (suspect) {
+            void this.#handleOutageSuspect(suspect);
+        }
+    }
     /**
      * Resolve the source label of the page being scraped so sub-resources
      * captured during its render can inherit the correct lineage label
@@ -540,6 +724,12 @@ class Crawler extends EventEmitter {
         // does not inherit "host alive" claims from a prior run that may have
         // happened on an entirely different network.
         this.#successfulHosts.clear();
+        // Network-outage state is per-crawl too: a sliding window of errors
+        // (or a gate left closed) from a prior run on this same `Crawler`
+        // instance must not leak into a fresh session. `#networkGate.open()`
+        // is a no-op if already open.
+        this.#networkOutageDetector.reset();
+        this.#networkGate.open();
         // external URL の追跡（target は deal の total/done から導出）
         const externalUrls = new Set();
         const externalDoneUrls = new Set();
@@ -587,6 +777,12 @@ class Crawler extends EventEmitter {
                 return Promise.all(ops).then(() => { });
             };
             return async () => {
+                // Pause here, not inside `fetchDestination` or deeper, so a
+                // paused worker shows as a long-running dealer task instead
+                // of requiring any change to `@d-zero/dealer` itself — a
+                // closed gate resolves the instant `#handleOutageSuspect`'s
+                // recovery probe succeeds (see `network-gate.ts`).
+                await this.#networkGate.wait();
                 // Interval delay is handled here instead of by dealer because
                 // DNS-burned hosts must skip the wait entirely. Spending the
                 // per-URL interval on a host the cache already knows is dead
@@ -741,6 +937,7 @@ class Crawler extends EventEmitter {
                     this.#handleResult(result, url, enqueue, paginationState, concurrency);
                     const parentSource = await this.#resolveParentSource(url);
                     this.#handleResources(result.resources, parentSource);
+                    this.#handleConsoleLogs(result.consoleLogs, url, result.pageData?.redirectPaths ?? []);
                     log(formatResultSummary(result));
                     // Phase errors must be emitted AFTER 'page' / 'externalPage'
                     // so the orchestrator's WriteQueue sees `setPage` before
@@ -814,6 +1011,52 @@ class Crawler extends EventEmitter {
         crawlerLog('Crawl End');
         void this.emit('crawlEnd', {});
     }
+    /**
+     * While {@link #networkGate} is closed, probe every
+     * `networkOutageProbeIntervalMs` until one succeeds, then reopen the
+     * gate and emit `networkOutageRecovered`.
+     *
+     * If the crawl is aborted while this loop is running, the gate is
+     * opened anyway (so any worker stuck on `#networkGate.wait()` can
+     * unblock and `deal()` can resolve) but `networkOutageRecovered` is NOT
+     * emitted — an abort says nothing about whether the network actually
+     * recovered, so the `network_outages` row is deliberately left open for
+     * the next writer session's boot-time finalizer
+     * (`close-stale-open-network-outages.ts`) to resolve. Either way,
+     * {@link #onGateReopened} still runs — the cached failures are stale
+     * regardless of why the gate reopened.
+     * @param probeHost - The hostname to probe, chosen once by
+     *   {@link #handleOutageSuspect} and reused for every attempt in this loop.
+     * @param startedAt - The outage's `startedAt`, forwarded to {@link #onGateReopened}.
+     */
+    async #runRecoveryProbeLoop(probeHost, startedAt) {
+        const probe = this.#options.networkProbe ?? probeNetwork;
+        const bailIfAborted = () => {
+            if (!this.#abortController.signal.aborted) {
+                return false;
+            }
+            this.#networkGate.open();
+            this.#onGateReopened(startedAt, Date.now());
+            return true;
+        };
+        if (bailIfAborted()) {
+            return;
+        }
+        for (;;) {
+            await delay(this.#options.networkOutageProbeIntervalMs);
+            if (bailIfAborted()) {
+                return;
+            }
+            const recovered = await probe(probeHost);
+            if (recovered) {
+                const endedAt = Date.now();
+                this.#networkGate.open();
+                this.#onGateReopened(startedAt, endedAt);
+                void this.emit('networkOutageRecovered', { endedAt });
+                return;
+            }
+        }
+    }
     /**
      * Orchestrates the full scrape pipeline for a single URL.
      *
@@ -874,6 +1117,7 @@ class Crawler extends EventEmitter {
                     type: 'success',
                     pageData: metadataOnly ? { ...pageData, isTarget: false } : pageData,
                     resources: [],
+                    consoleLogs: [],
                 };
             }
         }
@@ -1001,6 +1245,7 @@ class Crawler extends EventEmitter {
                 return {
                     type: 'error',
                     resources: [],
+                    consoleLogs: [],
                     error: {
                         name: error instanceof Error ? error.name : 'Error',
                         message: errorMessage,
@@ -1014,6 +1259,7 @@ class Crawler extends EventEmitter {
             return {
                 type: 'error',
                 resources: [],
+                consoleLogs: [],
                 error: {
                     name: error instanceof Error ? error.name : 'Error',
                     message: errorMessage,
@@ -1061,6 +1307,7 @@ class Crawler extends EventEmitter {
                         type: 'success',
                         pageData: { ...titleResult, isTarget: false },
                         resources: [],
+                        consoleLogs: [],
                     };
                 }
                 catch (error) {
@@ -1071,6 +1318,7 @@ class Crawler extends EventEmitter {
                 type: 'success',
                 pageData: { ...headCheckResult, isTarget: false },
                 resources: [],
+                consoleLogs: [],
             };
         }
         // Non-HTML content — skip browser
@@ -1080,6 +1328,7 @@ class Crawler extends EventEmitter {
                 type: 'success',
                 pageData: headCheckResult,
                 resources: [],
+                consoleLogs: [],
             };
         }
         // HTML or unknown content type — launch browser with preflight result.
@@ -1265,9 +1514,11 @@ class Crawler extends EventEmitter {
             retries: this.#options.retry,
             label: 'HEAD request',
             onWait: (determinedInterval, retryCount, label, error) => {
+                this.#recordNetworkError(error.message, host);
                 update(`${label}: ${error.message} — %countdown(${determinedInterval},fetchHead_${laneIndex}_${retryCount},s)%s (retry #${retryCount + 1})`);
             },
             onGiveUp: (retryCount, error, label) => {
+                this.#recordNetworkError(error.message, host);
                 // Burn the host so subsequent URLs short-circuit — but ONLY
                 // when this is the first time we've ever seen the host fail
                 // in this session. A host that responded earlier is treated
@@ -1283,6 +1534,12 @@ class Crawler extends EventEmitter {
                     successfulHosts: this.#successfulHosts,
                 })) {
                     dnsBurnedHostCache.set(host, 'dns');
+                    // Recorded so a later outage recovery can tell THIS
+                    // burn (possibly outage-caused) apart from a
+                    // preload-seeded one (a cross-session, confirmed-dead
+                    // verdict that must never be undone by an in-session
+                    // recovery) — see `evict-outage-tainted-dns-burns.ts`.
+                    dnsBurnedHostBurnTimestamps.set(host, Date.now());
                 }
                 update(c.red(`${label}: gave up after ${retryCount} retries — ${error.message}`));
             },
@@ -1355,8 +1612,8 @@ class Crawler extends EventEmitter {
             //    sub-resource requests issued from the same page —
             //    including cross-origin requests to a different hostname
             //    sharing the same IP / port (e.g. an embedded
-            //    `<img src="http://127.0.0.1:8010/…">` loaded from a
-            //    `localhost:8010` page) — get the cached `Authorization`
+            //    `<img src="http://127.0.0.1:PORT/…">` loaded from a
+            //    `localhost:PORT` page) — get the cached `Authorization`
             //    header re-attached by the network stack. The
             //    `Fetch.authRequired` event never fires for these
             //    pre-emptive attachments, so neither `page.authenticate`
@@ -1399,6 +1656,7 @@ class Crawler extends EventEmitter {
                 metadataOnly,
                 retries: this.#options.retry,
                 headCheckResult,
+                mainContentSelector: this.#options.mainContentSelector,
             });
             // Image dom-path capture runs here — after the scrape completed but
             // while `page` is still alive — because beholder's image metadata
@@ -1477,6 +1735,7 @@ class Crawler extends EventEmitter {
             return {
                 type: 'error',
                 resources: [],
+                consoleLogs: [],
                 error: {
                     name: error instanceof Error ? error.name : 'Error',
                     message: error instanceof Error ? error.message : String(error),

package/lib/crawler/dns-burned-host-burn-timestamps.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Companion to `dnsBurnedHostCache`: records the epoch-ms timestamp of each
+ * host burned by THIS session's `#sendHeadRequest` `onGiveUp` path (the
+ * `shouldBurnHost`-gated write in `crawler.ts`).
+ *
+ * Deliberately NOT populated by `#preloadDnsBurnedHostCache`'s seeding from
+ * a previous session's `crawl_errors` — those entries represent hosts
+ * already proven dead across sessions and must never be evicted just
+ * because THIS session's network happened to recover from an outage. Since
+ * `dnsBurnedHostCache`'s value (`ErrorKind`) carries no provenance of its
+ * own (preload-seeded and session-learned burns are indistinguishable by
+ * value alone), this separate timestamp map is what lets
+ * `evict-outage-tainted-dns-burns.ts` tell them apart: only hosts present
+ * HERE are eligible for eviction, and only if their burn timestamp falls
+ * inside the just-recovered outage's window.
+ */
+export declare const dnsBurnedHostBurnTimestamps: Map<string, number>;

package/lib/crawler/dns-burned-host-burn-timestamps.js

@@ -0,0 +1,17 @@
+/**
+ * Companion to `dnsBurnedHostCache`: records the epoch-ms timestamp of each
+ * host burned by THIS session's `#sendHeadRequest` `onGiveUp` path (the
+ * `shouldBurnHost`-gated write in `crawler.ts`).
+ *
+ * Deliberately NOT populated by `#preloadDnsBurnedHostCache`'s seeding from
+ * a previous session's `crawl_errors` — those entries represent hosts
+ * already proven dead across sessions and must never be evicted just
+ * because THIS session's network happened to recover from an outage. Since
+ * `dnsBurnedHostCache`'s value (`ErrorKind`) carries no provenance of its
+ * own (preload-seeded and session-learned burns are indistinguishable by
+ * value alone), this separate timestamp map is what lets
+ * `evict-outage-tainted-dns-burns.ts` tell them apart: only hosts present
+ * HERE are eligible for eviction, and only if their burn timestamp falls
+ * inside the just-recovered outage's window.
+ */
+export const dnsBurnedHostBurnTimestamps = new Map();

package/lib/crawler/evict-network-classified-destination-cache-entries.d.ts

@@ -0,0 +1,26 @@
+import type { PageData } from '@d-zero/beholder';
+/**
+ * Delete every `destinationCache` entry whose cached value is an `Error`
+ * that classifies as a {@link NETWORK_RELATED_ERROR_KINDS network-related}
+ * kind, leaving successes and site-specific failures (`tls`,
+ * `client-blocked`, `connection-refused`, …) untouched.
+ *
+ * Called by `Crawler` whenever its network gate transitions from closed to
+ * open (whether via a successful recovery probe or an abort), so a URL
+ * that failed with `getaddrinfo ENOTFOUND` only because the operator's own
+ * network was down does not stay wrongly cached as "this host is dead" for
+ * the rest of the session.
+ *
+ * Deliberately does NOT distinguish "cached during THIS specific outage"
+ * from "cached during an earlier blip this session" — any cached error
+ * whose KIND looks network-related is, by definition, potentially stale
+ * evidence about the operator's network rather than the target site, so it
+ * is always safe to re-test after any recovery. The trade-off is a few
+ * redundant HEAD attempts for hosts that were already dead independent of
+ * the outage; the alternative (a stale "outage-tainted" failure verdict
+ * surviving the rest of the session) is strictly worse.
+ * @param cache - The cache to sweep. Takes the `Map` explicitly (rather
+ *   than importing the `destinationCache` singleton directly) so this stays
+ *   unit-testable without touching module-level state.
+ */
+export declare function evictNetworkClassifiedDestinationCacheEntries(cache: Map<string, PageData | Error>): void;

package/lib/crawler/evict-network-classified-destination-cache-entries.js

@@ -0,0 +1,34 @@
+import { classifyErrorKind } from '../classify-error-kind.js';
+import { NETWORK_RELATED_ERROR_KINDS } from '../network-related-error-kinds.js';
+/**
+ * Delete every `destinationCache` entry whose cached value is an `Error`
+ * that classifies as a {@link NETWORK_RELATED_ERROR_KINDS network-related}
+ * kind, leaving successes and site-specific failures (`tls`,
+ * `client-blocked`, `connection-refused`, …) untouched.
+ *
+ * Called by `Crawler` whenever its network gate transitions from closed to
+ * open (whether via a successful recovery probe or an abort), so a URL
+ * that failed with `getaddrinfo ENOTFOUND` only because the operator's own
+ * network was down does not stay wrongly cached as "this host is dead" for
+ * the rest of the session.
+ *
+ * Deliberately does NOT distinguish "cached during THIS specific outage"
+ * from "cached during an earlier blip this session" — any cached error
+ * whose KIND looks network-related is, by definition, potentially stale
+ * evidence about the operator's network rather than the target site, so it
+ * is always safe to re-test after any recovery. The trade-off is a few
+ * redundant HEAD attempts for hosts that were already dead independent of
+ * the outage; the alternative (a stale "outage-tainted" failure verdict
+ * surviving the rest of the session) is strictly worse.
+ * @param cache - The cache to sweep. Takes the `Map` explicitly (rather
+ *   than importing the `destinationCache` singleton directly) so this stays
+ *   unit-testable without touching module-level state.
+ */
+export function evictNetworkClassifiedDestinationCacheEntries(cache) {
+    for (const [key, value] of cache) {
+        if (value instanceof Error &&
+            NETWORK_RELATED_ERROR_KINDS.has(classifyErrorKind(value.message))) {
+            cache.delete(key);
+        }
+    }
+}

package/lib/crawler/evict-outage-tainted-dns-burns.d.ts

@@ -0,0 +1,23 @@
+import type { OutageWindow } from '../is-within-outage-window.js';
+import type { ErrorKind } from '../types.js';
+/**
+ * Undo `dnsBurnedHostCache` burns whose recorded timestamp (see
+ * `dns-burned-host-burn-timestamps.ts`) falls inside `window` — i.e. hosts
+ * THIS session burned because a HEAD request failed with a
+ * `dns`-classified error while (or just before) the operator's own network
+ * was down, not because the host is actually dead.
+ *
+ * Preload-seeded burns are structurally immune: they never appear in
+ * `burnTimestamps` (see that module's docstring), so this function can
+ * never touch them regardless of the window.
+ * @param options - Named parameters.
+ * @param options.cache - `dnsBurnedHostCache` (or a test double with the same shape).
+ * @param options.burnTimestamps - `dnsBurnedHostBurnTimestamps` (or a test double).
+ * @param options.window - The just-resolved outage window (`{ startedAt, endedAt }`)
+ *   to test each burn's timestamp against.
+ */
+export declare function evictOutageTaintedDnsBurns(options: {
+    cache: Map<string, ErrorKind>;
+    burnTimestamps: Map<string, number>;
+    window: OutageWindow;
+}): void;

package/lib/crawler/evict-outage-tainted-dns-burns.js

@@ -0,0 +1,26 @@
+import { isWithinOutageWindow } from '../is-within-outage-window.js';
+/**
+ * Undo `dnsBurnedHostCache` burns whose recorded timestamp (see
+ * `dns-burned-host-burn-timestamps.ts`) falls inside `window` — i.e. hosts
+ * THIS session burned because a HEAD request failed with a
+ * `dns`-classified error while (or just before) the operator's own network
+ * was down, not because the host is actually dead.
+ *
+ * Preload-seeded burns are structurally immune: they never appear in
+ * `burnTimestamps` (see that module's docstring), so this function can
+ * never touch them regardless of the window.
+ * @param options - Named parameters.
+ * @param options.cache - `dnsBurnedHostCache` (or a test double with the same shape).
+ * @param options.burnTimestamps - `dnsBurnedHostBurnTimestamps` (or a test double).
+ * @param options.window - The just-resolved outage window (`{ startedAt, endedAt }`)
+ *   to test each burn's timestamp against.
+ */
+export function evictOutageTaintedDnsBurns(options) {
+    const { cache, burnTimestamps, window } = options;
+    for (const [host, burnedAt] of burnTimestamps) {
+        if (isWithinOutageWindow(burnedAt, [window])) {
+            cache.delete(host);
+            burnTimestamps.delete(host);
+        }
+    }
+}

package/lib/crawler/fetch-destination.js

@@ -196,6 +196,8 @@ async function _fetchHead(url, isExternal, method, titleBytesLimit, userAgent, t
                     imageList: [],
                     anchorList: [],
                     html: '',
+                    mainContents: null,
+                    scrollHeight: null,
                     isSkipped: false,
                 };
             };

package/lib/crawler/link-to-page-data.js

@@ -48,6 +48,8 @@ export function linkToPageData(link) {
         anchorList: [],
         imageList: [],
         html: '',
+        mainContents: null,
+        scrollHeight: null,
         isSkipped: false,
     };
 }

package/lib/crawler/network-gate.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Level-triggered gate that dealer worker callbacks await before doing any
+ * network work. Achieves "pause the crawl until the network recovers"
+ * without touching `@d-zero/dealer` internals — a closed gate just makes
+ * the worker's task body a long-running `await`, which is all dealer ever
+ * sees.
+ *
+ * **`epoch`** counts outage generations: it increments on every `close()`
+ * transition (open → closed) and holds that value for the remainder of the
+ * closed period AND through the following open period, until the next
+ * `close()`. This is the hook `destinationCache` / `dnsBurnedHostCache`
+ * eviction uses: an entry written while the gate was closed is tagged with
+ * `gate.epoch` at write time; after `open()`, the caller purges entries
+ * tagged with that same epoch value. Because `epoch` only advances on
+ * `close()` (never on `open()`), entries written during ordinary (open)
+ * operation are never mistakenly tagged with a soon-to-be-purged epoch —
+ * only writes that happen while `isOpen` is `false` get tagged at all.
+ */
+export default class NetworkGate {
+    #private;
+    /** `true` when the gate is open (the default). */
+    get isOpen(): boolean;
+    /**
+     * Current outage-generation counter. See the class docstring for how
+     * cache-eviction callers are expected to use this value.
+     */
+    get epoch(): number;
+    /**
+     * Close the gate, blocking future {@link wait} callers until the next
+     * {@link open}. Idempotent — calling `close()` while already closed does
+     * nothing (in particular, it does NOT bump {@link epoch} a second time,
+     * which would otherwise let a still-open outage's cache entries escape
+     * eviction under a stale epoch number).
+     */
+    close(): void;
+    /**
+     * Open the gate, resolving every {@link wait} caller currently pending —
+     * including ones that subscribed after `close()` but before this call.
+     * Idempotent — calling `open()` while already open does nothing.
+     */
+    open(): void;
+    /**
+     * Resolve immediately if the gate is open; otherwise resolve on the next
+     * {@link open}. Multiple concurrent callers all resolve on the same
+     * `open()` call — no awaiter is dropped regardless of when it subscribed
+     * relative to others.
+     */
+    wait(): Promise<void>;
+}