@nitpicker/crawler 0.14.0 → 0.16.0

package/lib/crawler-orchestrator.js

@@ -13,6 +13,7 @@ import { dnsBurnedHostCache } from './crawler/dns-burned-host-cache.js';
 import { dnsBurnedHostShortCircuitCounter } from './crawler/dns-burned-host-short-circuit-counter.js';
 import { findScopeEntry } from './crawler/find-scope-entry.js';
 import { isLikelyHtmlUrl } from './crawler/is-likely-html-url.js';
+import { networkOutageSummaryCounter } from './crawler/network-outage-summary-counter.js';
 import { PreloadShortCircuitError } from './crawler/preload-short-circuit-error.js';
 import { protocolAgnosticKey } from './crawler/protocol-agnostic-key.js';
 import { crawlerLog, log } from './debug.js';
@@ -63,8 +64,28 @@ export class CrawlerOrchestrator extends EventEmitter {
     #archive;
     /** The crawler engine that discovers and scrapes pages. */
     #crawler;
+    /**
+     * `dedupe_cap_events.id` for each shape confirmed capped this session, so
+     * `crawlEnd` can look up the right row to finalize with
+     * `Crawler#getDedupeCapRejections`'s counts. A `Map` (not a single
+     * scalar like {@link #openNetworkOutageId}) because, unlike a network
+     * outage, more than one shape can be capped simultaneously within one
+     * crawl.
+     */
+    #dedupeCapEventIds = new Map();
     /** Whether the crawl was started from a pre-defined URL list (non-recursive mode). */
     #fromList;
+    /**
+     * The `network_outages` row id for the currently-open outage, or `null`
+     * when none is open. Set by the `networkOutageConfirmed` handler (once
+     * the INSERT resolves) and consumed by `networkOutageRecovered` — the
+     * `Crawler` class never touches the archive itself and has no way to
+     * know the row's id, so the orchestrator is the only place that can
+     * bridge the two events for the same outage.
+     */
+    #openNetworkOutageId = null;
+    /** `startedAt` of the currently-open outage, tracked alongside {@link #openNetworkOutageId} so `networkOutageRecovered` can compute a duration for {@link networkOutageSummaryCounter}. */
+    #openNetworkOutageStartedAt = null;
     /** Serializes archive writes from crawler event handlers (FIFO). */
     #writeQueue = new WriteQueue();
     /**
@@ -138,6 +159,21 @@ export class CrawlerOrchestrator extends EventEmitter {
             // rows continue to land in pages/resources with the DB DEFAULT
             // `'crawled'` provenance label.
             inventoryMode: options?.inventoryMode ?? null,
+            // Forwarded as-is (including `undefined`) — `Crawler`'s own
+            // constructor merges each against its `DEFAULT_NETWORK_OUTAGE_*`
+            // constant, so omitting them here is exactly "use the default".
+            networkOutageWindowMs: options?.networkOutageWindowMs,
+            networkOutageErrorThreshold: options?.networkOutageErrorThreshold,
+            networkOutageHostThreshold: options?.networkOutageHostThreshold,
+            networkOutageProbeIntervalMs: options?.networkOutageProbeIntervalMs,
+            networkProbe: options?.networkProbe ?? null,
+            dedupeCap: options?.dedupeCap ?? null,
+            dedupeMapCap: options?.dedupeMapCap,
+            // Only the four resuming-session static methods
+            // (`append`/`inventory`/`retryFailed`/`resume`) pass this — a
+            // fresh `crawling()` has no archive history to seed from (see
+            // `CrawlConfig.preloadedStickyShapeKeys`'s JSDoc).
+            preloadedStickyShapeKeys: options?.preloadedStickyShapeKeys ?? [],
         });
     }
     /**
@@ -166,6 +202,11 @@ export class CrawlerOrchestrator extends EventEmitter {
      */
     async crawling(list, opts) {
         const writeQueue = this.#writeQueue;
+        // Per-session state, like `Crawler`'s own `#successfulHosts.clear()` /
+        // `#networkGate.open()` reset at the start of `#runDeal` — a fresh
+        // session must not inherit a dangling outage id from a prior one.
+        this.#openNetworkOutageId = null;
+        this.#openNetworkOutageStartedAt = null;
         return new Promise((resolve, reject) => {
             this.#crawler.on('error', (error) => {
                 if (error.error instanceof PreloadShortCircuitError) {
@@ -212,6 +253,84 @@ export class CrawlerOrchestrator extends EventEmitter {
                     .catch((error) => reject(error));
                 void this.emit('redirect', { result });
             });
+            this.#crawler.on('networkOutageConfirmed', ({ startedAt, detectedAt, probeHost, triggerErrorCount, triggerHostCount }) => {
+                crawlerLog('Network outage confirmed: probeHost=%s triggerErrorCount=%d triggerHostCount=%d', probeHost, triggerErrorCount, triggerHostCount);
+                // event notice; mirrors `#finalizeCrawlSession`'s unconditional
+                // `console.error` for the DNS-burn short-circuit summary.
+                console.error(`[network] outage suspected — pausing workers (probe host: ${probeHost ?? 'none'})`);
+                writeQueue
+                    .enqueue(async () => {
+                    // Both fields are set together, inside this single
+                    // closure, so the pair can never fall out of sync
+                    // (e.g. one set synchronously above while the other
+                    // waits on the INSERT) — `networkOutageRecovered`'s
+                    // queued closure always sees either both set or
+                    // neither.
+                    const id = await this.#archive.insertNetworkOutage({
+                        startedAt,
+                        detectedAt,
+                        probeHost,
+                        triggerErrorCount,
+                        triggerHostCount,
+                    });
+                    this.#openNetworkOutageId = id;
+                    this.#openNetworkOutageStartedAt = startedAt;
+                })
+                    .catch((error) => reject(error));
+            });
+            this.#crawler.on('networkOutageRecovered', ({ endedAt }) => {
+                // The `id` read is deferred to INSIDE the queued closure, not
+                // read synchronously here, because `networkOutageConfirmed`'s
+                // INSERT is itself only queued (not awaited) when that event
+                // fires — `#openNetworkOutageId` is not guaranteed to be set
+                // yet at the instant `networkOutageRecovered` fires (the two
+                // events can arrive in quick succession, e.g. in tests that
+                // drive them back-to-back with no real probe-interval delay
+                // between them). `WriteQueue` runs enqueued operations in
+                // submission order, so by the time THIS closure actually
+                // executes, the confirm's INSERT closure (enqueued first) has
+                // already completed and `#openNetworkOutageId` is reliably set.
+                writeQueue
+                    .enqueue(() => {
+                    const id = this.#openNetworkOutageId;
+                    const startedAt = this.#openNetworkOutageStartedAt;
+                    if (id === null) {
+                        // Defensive: `networkOutageConfirmed` always
+                        // precedes `networkOutageRecovered` on the same
+                        // `Crawler` instance. If this fires anyway, there
+                        // is no row to close.
+                        crawlerLog('Network outage recovered but no open outage id was tracked');
+                        return Promise.resolve();
+                    }
+                    this.#openNetworkOutageId = null;
+                    this.#openNetworkOutageStartedAt = null;
+                    const durationMs = endedAt - (startedAt ?? endedAt);
+                    networkOutageSummaryCounter.confirmedCount++;
+                    networkOutageSummaryCounter.totalDurationMs += durationMs;
+                    crawlerLog('Network outage recovered: id=%d endedAt=%d', id, endedAt);
+                    // eslint-disable-next-line no-console -- see the confirmed handler above
+                    console.error(`[network] recovered after ${Math.round(durationMs / 1000)}s`);
+                    return this.#archive.closeNetworkOutage(id, endedAt);
+                })
+                    .catch((error) => reject(error));
+            });
+            this.#crawler.on('dedupeCap', ({ shapeKey, sampleUrl, bodyHash, effectiveThreshold, observedCount }) => {
+                crawlerLog('Dedupe cap reached: shapeKey=%s effectiveThreshold=%d observedCount=%d', shapeKey, effectiveThreshold, observedCount);
+                console.error(`[dedupe-cap] same-cluster trap confirmed: ${shapeKey} (sample: ${sampleUrl})`);
+                writeQueue
+                    .enqueue(async () => {
+                    const id = await this.#archive.insertDedupeCapEvent({
+                        shapeKey,
+                        sampleUrl,
+                        bodyHash,
+                        effectiveThreshold,
+                        observedCount,
+                        detectedAt: Date.now(),
+                    });
+                    this.#dedupeCapEventIds.set(shapeKey, id);
+                })
+                    .catch((error) => reject(error));
+            });
             this.#crawler.on('response', ({ resource, source }) => {
                 writeQueue
                     .enqueue(() => this.#archive.setResources(resource, source))
@@ -222,7 +341,57 @@ export class CrawlerOrchestrator extends EventEmitter {
                     .enqueue(() => this.#archive.setResourcesReferrers(resource))
                     .catch((error) => reject(error));
             });
+            this.#crawler.on('consoleLogs', ({ pageUrl, redirectPaths, entries }) => {
+                writeQueue
+                    .enqueue(() => this.#archive.setConsoleLogs(pageUrl, redirectPaths, entries))
+                    .catch((error) => reject(error));
+            });
             this.#crawler.on('crawlEnd', () => {
+                // Deferred to INSIDE a queued closure, not read synchronously
+                // here, for the same reason `networkOutageRecovered`'s handler
+                // defers reading `#openNetworkOutageId`: a `dedupeCap` event's
+                // INSERT closure may still be queued (not yet executed) at the
+                // instant `crawlEnd` fires. `WriteQueue` runs enqueued
+                // operations in submission order, so by the time THIS closure
+                // executes, every earlier-queued `dedupeCap` INSERT has
+                // already completed and `#dedupeCapEventIds` is reliably
+                // populated.
+                writeQueue
+                    .enqueue(async () => {
+                    const rejections = this.#crawler.getDedupeCapRejections();
+                    // Finalize every shape capped THIS session (has an id in
+                    // `#dedupeCapEventIds`), not just the ones with a nonzero
+                    // rejection count — a shape that capped near the end of the
+                    // crawl (or whose remaining anchors all happened to be
+                    // discovered before it capped) never enters `rejections` at
+                    // all, and would otherwise stay `rejected_count: NULL` forever
+                    // despite the crawl completing normally, corrupting the "NULL
+                    // means the crawl never reached crawlEnd" contract
+                    // `list-dedupe-cap-events.ts` documents.
+                    const shapeKeysToFinalize = new Set([
+                        ...this.#dedupeCapEventIds.keys(),
+                        ...rejections.keys(),
+                    ]);
+                    await Promise.all([...shapeKeysToFinalize].map((shapeKey) => {
+                        const rejectedCount = rejections.get(shapeKey) ?? 0;
+                        const id = this.#dedupeCapEventIds.get(shapeKey);
+                        // A shape capped THIS session has an id here (the
+                        // `dedupeCap` event always enqueues an INSERT before any
+                        // rejection for that shape can be counted) and is
+                        // finalized once via its row id. A shape with no id was
+                        // never observed this session at all — it was preloaded
+                        // into `DedupeCapTracker`'s sticky set from an EARLIER
+                        // session's `dedupe_cap_events` row (see
+                        // `CrawlConfig.preloadedStickyShapeKeys`'s JSDoc), so gate
+                        // rejections still accumulate for it but no new row (and
+                        // thus no id) is ever created. That earlier row's count is
+                        // accumulated onto by shape_key instead of overwritten.
+                        return id === undefined
+                            ? this.#archive.accumulateDedupeCapRejectedCount(shapeKey, rejectedCount)
+                            : this.#archive.finalizeDedupeCapEvent(id, rejectedCount);
+                    }));
+                })
+                    .catch((error) => reject(error));
                 writeQueue
                     .drain()
                     .then(() => resolve())
@@ -387,7 +556,13 @@ export class CrawlerOrchestrator extends EventEmitter {
         const absFilePath = path.isAbsolute(archivePath)
             ? archivePath
             : path.resolve(cwd, archivePath);
-        const archive = await Archive.open({ filePath: absFilePath, cwd });
+        // See `ArchiveOpenOptions.openPluginData` for why this must be `true`
+        // on every writer path that calls `write()`.
+        const archive = await Archive.open({
+            filePath: absFilePath,
+            cwd,
+            openPluginData: true,
+        });
         // Any throw between here and the successful return must release the
         // archive lock and clean up tmpDir; the caller's `close()` only runs on
         // the happy path. Errors from `close()` itself are intentionally
@@ -424,9 +599,14 @@ export class CrawlerOrchestrator extends EventEmitter {
                     scopeMap.set(parsed.hostname, [...existing, parsed]);
                 }
                 await archive.repromoteExternalPages(scopeMap, archived);
+                // Seed the sticky set from prior sessions' confirmed traps so
+                // `--append` does not pay the cost of re-discovering them (see
+                // `DedupeCapTracker`'s constructor JSDoc).
+                const preloadedStickyShapeKeys = await archive.listDedupeCapShapeKeys();
                 const orchestrator = new CrawlerOrchestrator(archive, {
                     ...mergedConfig,
                     roots: mergedRoots,
+                    preloadedStickyShapeKeys,
                 });
                 const { scraped, pending } = await archive.getCrawlingState();
                 const resources = await archive.getResourceUrlList();
@@ -481,22 +661,29 @@ export class CrawlerOrchestrator extends EventEmitter {
      * 3. Reject archives with unfinished `pending` URLs — those would inherit
      *    the inventory `source` label by mistake. Operator must resume /
      *    retry-failed first.
-     * 4. Parse the URL list. Anything outside the archived scope is warned
-     *    and skipped (inventory is per-server by design).
-     * 5. Subtract URLs that already exist in `pages` or `resources` so the
+     * 4. If `source` is given, archive its exact bytes under
+     *    `inventory/<sha256>.txt` (see {@link Archive.saveInventorySourceList}).
+     *    Done before scope classification so even a run that discards every
+     *    URL (out of scope or already known) still leaves a recoverable
+     *    copy of what was fed in.
+     * 5. Parse the URL list — the CLI has already warned-and-dropped
+     *    unparseable-URL lines before calling this method, so every
+     *    remaining entry parses. Anything outside the archived scope is
+     *    warned and skipped (inventory is per-server by design).
+     * 6. Subtract URLs that already exist in `pages` or `resources` so the
      *    second (and N-th) inventory pass is a no-op for known rows — keeps
      *    `'inventory-seed'` rows from being silently demoted.
-     * 6. Make `<archive>.bak`. Anything thrown beyond this point restores
+     * 7. Make `<archive>.bak`. Anything thrown beyond this point restores
      *    from the backup.
-     * 7. HEAD-probe each novel URL. Responses classified as HTML are queued
+     * 8. HEAD-probe each novel URL. Responses classified as HTML are queued
      *    as Crawler seeds (`'inventory-seed'`); everything else is recorded
      *    in `resources` directly as `'inventory-seed'` (no browser launch).
-     * 8. If any HTML seeds exist, start a Crawler with
+     * 9. If any HTML seeds exist, start a Crawler with
      *    `inventoryMode = { seedUrls }` so the rendered page and every newly
      *    discovered downstream link is labelled correctly. `resume` is fed
      *    the existing `scraped` / `resources` sets so links into already-
      *    crawled pages stop at the seen-gate without re-rendering.
-     * 9. Drop the backup on success; restore it on any throw.
+     * 10. Drop the backup on success; restore it on any throw.
      *
      * Mutually exclusive with `--append` / `--retry-failed` / `--resume` /
      * `--diff` / `--list` / `--list-file` / `--single` / `--output` — the
@@ -506,20 +693,22 @@ export class CrawlerOrchestrator extends EventEmitter {
      * @param inventoryUrls - Pre-read URL list (one URL per element).
      * @param options - Optional config overrides — most callers leave this blank and let the archived config flow through.
      * @param initializedCallback - Hook invoked once the orchestrator is constructed but before `crawling` runs (the CLI uses it to attach progress reporting).
-     * @param sourceFileSha256 - **Pre-computed** SHA-256 hex digest of the
-     *   source URL list. The orchestrator deliberately does NOT receive
-     *   the file path: the path is privacy-sensitive (leaks user-home /
-     *   OS structure when archives are shared) and we want it lifted off
-     *   this boundary so no future log line / breadcrumb / error message
-     *   inside the orchestrator can accidentally re-leak it. The CLI
-     *   computes the digest via `computeFileSha256(resolvedListFile)`
-     *   and passes it through here. Pass `null` for programmatic
-     *   callers that built `inventoryUrls` in-memory; the audit row's
-     *   `source_file_sha256` column will be `NULL`.
+     * @param source - The CLI's already-read source list, as `{ sha256, bytes }`.
+     *   The orchestrator deliberately does NOT receive the file path: the
+     *   path is privacy-sensitive (leaks user-home / OS structure when
+     *   archives are shared) and we want it lifted off this boundary so no
+     *   future log line / breadcrumb / error message inside the orchestrator
+     *   can accidentally re-leak it. `bytes` is archived verbatim under
+     *   `inventory/<sha256>.txt` (see {@link Archive.saveInventorySourceList})
+     *   before scope classification, so a later `--inventory` run against
+     *   the same list is an audit no-op even when it discards zero new
+     *   URLs. Pass `null` for programmatic callers that built
+     *   `inventoryUrls` in-memory; the audit row's `source_file_sha256`
+     *   column will be `NULL` and no source list is archived.
      * @returns The orchestrator instance after a successful inventory pass.
      * @throws {Error} When `inventoryUrls` is empty, the archive is in list mode, or pending URLs from a previous crawl remain unresolved.
      */
-    static async inventory(archivePath, inventoryUrls, options, initializedCallback, sourceFileSha256 = null) {
+    static async inventory(archivePath, inventoryUrls, options, initializedCallback, source = null) {
         if (inventoryUrls.length === 0) {
             throw new Error('inventory: URL list is empty');
         }
@@ -527,7 +716,13 @@ export class CrawlerOrchestrator extends EventEmitter {
         const absFilePath = path.isAbsolute(archivePath)
             ? archivePath
             : path.resolve(cwd, archivePath);
-        const archive = await Archive.open({ filePath: absFilePath, cwd });
+        // See `ArchiveOpenOptions.openPluginData` for why this must be `true`
+        // on every writer path that calls `write()`.
+        const archive = await Archive.open({
+            filePath: absFilePath,
+            cwd,
+            openPluginData: true,
+        });
         try {
             const archived = await archive.getConfig();
             if (archived.fromList) {
@@ -549,6 +744,14 @@ export class CrawlerOrchestrator extends EventEmitter {
                 // eslint-disable-next-line no-console -- operator-facing warning, must be visible regardless of DEBUG filters
                 console.warn(`inventory: archive has ${pending.length} pending URLs from a previous crawl. Proceeding — crawled-wins priority keeps their labels stable. Consider \`--resume\` first if you want the prior work finalized.`);
             }
+            // Archive the exact source bytes before scope classification, so
+            // even a run that discards every URL (out-of-scope or already
+            // known — see `novelUrls.length === 0` below) still leaves a
+            // recoverable copy of what was fed in. Content-hash-named, so a
+            // second `--inventory` pass with the same list is a no-op write.
+            if (source) {
+                await archive.saveInventorySourceList(source.sha256, source.bytes);
+            }
             // Parse + scope-classify the candidate URLs. sortUrl drops
             // unparseable strings; findScopeEntry separates in-scope from
             // out-of-scope.
@@ -703,7 +906,8 @@ export class CrawlerOrchestrator extends EventEmitter {
                     htmlSeedsCount: htmlSeeds.length,
                     nonHtmlCount: nonHtmlSeeds.length,
                     outOfScope,
-                    sourceFileSha256,
+                    sourceFileSha256: source?.sha256 ?? null,
+                    invalidSkipped: source?.invalidLineCount ?? null,
                 });
                 // Ingestion's DB writes are now committed. From here on a
                 // throw must NOT trigger the `.bak` restore (it would wipe
@@ -735,6 +939,16 @@ export class CrawlerOrchestrator extends EventEmitter {
                     inventoryMode: { seedUrls: seedSet },
                 };
                 if (htmlSeeds.length > 0) {
+                    // Seed the sticky set from prior sessions' confirmed traps
+                    // so `--inventory` does not pay the cost of
+                    // re-discovering them (see `DedupeCapTracker`'s
+                    // constructor JSDoc). Scoped to this branch only,
+                    // matching `#preloadDnsBurnedHostCache`'s scoping below —
+                    // the fallback (non-HTML-only) branch never calls
+                    // `orchestrator.crawling(...)`, so the tracker is never
+                    // consulted there.
+                    orchestratorOptions.preloadedStickyShapeKeys =
+                        await archive.listDedupeCapShapeKeys();
                     const orchestrator = new CrawlerOrchestrator(archive, orchestratorOptions);
                     // Re-read pending *after* the pre-insert so the strict-
                     // pending set includes the freshly inserted
@@ -853,7 +1067,13 @@ export class CrawlerOrchestrator extends EventEmitter {
         const absFilePath = path.isAbsolute(archivePath)
             ? archivePath
             : path.resolve(cwd, archivePath);
-        const archive = await Archive.open({ filePath: absFilePath, cwd });
+        // See `ArchiveOpenOptions.openPluginData` for why this must be `true`
+        // on every writer path that calls `write()`.
+        const archive = await Archive.open({
+            filePath: absFilePath,
+            cwd,
+            openPluginData: true,
+        });
         // Any throw between here and the successful return must release the
         // archive lock and clean up tmpDir; the caller's `close()` only runs on
         // the happy path.
@@ -880,7 +1100,14 @@ export class CrawlerOrchestrator extends EventEmitter {
                 log('Start retrying failed pages');
                 log('Archive %s', absFilePath);
                 log('Reset %d failed page(s)', resetUrls.length);
-                const orchestrator = new CrawlerOrchestrator(archive, config);
+                // Seed the sticky set from prior sessions' confirmed traps so
+                // `--retry-failed` does not pay the cost of re-discovering
+                // them (see `DedupeCapTracker`'s constructor JSDoc).
+                const preloadedStickyShapeKeys = await archive.listDedupeCapShapeKeys();
+                const orchestrator = new CrawlerOrchestrator(archive, {
+                    ...config,
+                    preloadedStickyShapeKeys,
+                });
                 const { scraped, pending } = await archive.getCrawlingState();
                 const resources = await archive.getResourceUrlList();
                 const pagesScrapedOffset = await archive.getScrapedHtmlPageCount();
@@ -929,9 +1156,14 @@ export class CrawlerOrchestrator extends EventEmitter {
     static async resume(stubPath, options, initializedCallback) {
         const archive = await Archive.resume(stubPath);
         const archivedConfig = await archive.getConfig();
+        // Seed the sticky set from prior sessions' confirmed traps so
+        // `--resume` does not pay the cost of re-discovering them (see
+        // `DedupeCapTracker`'s constructor JSDoc).
+        const preloadedStickyShapeKeys = await archive.listDedupeCapShapeKeys();
         const config = {
             ...archivedConfig,
             ...cleanObject(options),
+            preloadedStickyShapeKeys,
         };
         const orchestrator = new CrawlerOrchestrator(archive, config);
         const _url = await archive.getUrl();
@@ -990,10 +1222,10 @@ export class CrawlerOrchestrator extends EventEmitter {
      * no `--label` flag, so this is always the auto form.
      * `source_file_sha256` arrives pre-computed via
      * `aggregates.sourceFileSha256` (the CLI's `inventoryCrawl` ran
-     * `computeFileSha256` against the input txt before the orchestrator
-     * was even invoked). The orchestrator boundary deliberately never sees
-     * the absolute path — see {@link InventoryRunAggregates} for the
-     * privacy rationale.
+     * `computeFileSha256` against the bytes it read from the input txt,
+     * before the orchestrator was even invoked). The orchestrator boundary
+     * deliberately never sees the absolute path — see
+     * {@link InventoryRunAggregates} for the privacy rationale.
      *
      * **Audit-write failures abort the ingestion phase.** Swallowing them
      * would only be justified if the audit were the last write after the
@@ -1020,6 +1252,7 @@ export class CrawlerOrchestrator extends EventEmitter {
             new_pages: aggregates.htmlSeedsCount,
             new_resources: aggregates.nonHtmlCount,
             scope_skipped: aggregates.outOfScope,
+            invalid_skipped: aggregates.invalidSkipped,
         });
     }
     /**
@@ -1034,6 +1267,13 @@ export class CrawlerOrchestrator extends EventEmitter {
             // eslint-disable-next-line no-console
             console.error(`[preload] Short-circuited ${skipped} URL(s) on DNS-burned hosts`);
         }
+        const { confirmedCount, totalDurationMs } = networkOutageSummaryCounter;
+        if (confirmedCount > 0) {
+            // eslint-disable-next-line no-console
+            console.error(`[network] ${confirmedCount} outage(s), ${Math.round(totalDurationMs / 1000)}s total`);
+        }
+        networkOutageSummaryCounter.confirmedCount = 0;
+        networkOutageSummaryCounter.totalDurationMs = 0;
         clearDestinationCache();
         clearDnsBurnedHostCache();
     }

package/lib/crawler.d.ts

@@ -16,16 +16,37 @@ export { default as ArchiveResource } from './archive/resource.js';
 export * from './archive/types.js';
 export { default as Archive } from './archive/archive.js';
 export { isArchiveCacheDisabled } from './archive/cache/is-archive-cache-disabled.js';
+export { getArchiveCacheRoot } from './archive/cache/get-archive-cache-root.js';
+export { computeArchiveCacheKey } from './archive/cache/compute-archive-cache-key.js';
+export { resolveArchiveCacheDir } from './archive/cache/resolve-archive-cache-dir.js';
+export { listArchiveCacheEntries } from './archive/cache/list-archive-cache-entries.js';
+export { clearArchiveCacheRoot } from './archive/cache/clear-archive-cache-root.js';
+export { clearArchiveCacheEntry } from './archive/cache/clear-archive-cache-entry.js';
+export type { ArchiveCacheEntry, ArchiveCacheEntryKind } from './archive/cache/types.js';
 export { acquireArchiveLock, ArchiveLockError } from './archive/archive-lock.js';
 export { peekArchiveLockHolder } from './archive/peek-archive-lock.js';
 export type { ArchiveLockHolder } from './archive/peek-archive-lock.js';
 export type { FlatPageMetaColumns, JsonLdRow, JsonLdRowForInsert, TagRow, TagRowForInsert, JsonLdSummary, TagsSummary, TagInventoryEntry, PageDenormalizedColumns, } from './archive/meta/types.js';
 export { IncompatibleArchiveError } from './archive/meta/types.js';
 export { REQUIRED_FORMAT_VERSION } from './archive/meta/assert-compatible-version.js';
+export { computeBodyHash } from './archive/body-hash/compute-body-hash.js';
+export { decodeStoredBlob } from './archive/decode-html-blob.js';
+export { computeTierAAliasKey } from './archive/url-alias/compute-tier-a-alias-key.js';
+export { computeTierBAliasKey } from './archive/url-alias/compute-tier-b-alias-key.js';
 export { DEFAULT_EXCLUDED_EXTERNAL_URLS, CrawlerOrchestrator, } from './crawler-orchestrator.js';
 export * from './types.js';
 export * from './crawler/types.js';
 export { classifyErrorKind } from './classify-error-kind.js';
+export { NETWORK_RELATED_ERROR_KINDS } from './network-related-error-kinds.js';
+export type { OutageWindow } from './is-within-outage-window.js';
+export { isWithinOutageWindow } from './is-within-outage-window.js';
+export { default as NetworkOutageDetector } from './crawler/network-outage-detector.js';
+export { default as NetworkGate } from './crawler/network-gate.js';
+export type { NetworkProbe } from './crawler/probe-network.js';
+export { probeNetwork } from './crawler/probe-network.js';
+export { computeOutageClampTimestamp } from './archive/db-ops/outages/compute-outage-clamp-timestamp.js';
+export { chooseProbeHost } from './crawler/choose-probe-host.js';
+export { assertChromeIsInstalled } from './crawler/assert-chrome-installed.js';
 export { computeFileSha256 } from './utils/compute-file-sha256.js';
 export { populateEntityTables } from './archive/populate-entity-tables/populate-entities.js';
 export type { PageDomPathResolver } from './archive/populate-entity-tables/populate-image-items.js';
@@ -38,3 +59,4 @@ export { populateBlobRefs } from './archive/populate-ref-tables/populate-blob-re
 export { populateHeaderTables } from './archive/populate-ref-tables/populate-header-tables.js';
 export { loadResponseHeadersBySetIds } from './archive/db-ops/_shared/load-response-headers-by-set-ids.js';
 export { decodeJsonRef } from './archive/db-ops/_shared/decode-json-ref.js';
+export type { TemplateClusterBlockingEvidence, TemplateClusterBlockingReason, TemplateClusterLandmarkProfile, TemplateClusterLandmarkType, TemplateClusterReason, } from './archive/db-ops/analysis/types.js';

package/lib/crawler.js

@@ -17,15 +17,33 @@ export { default as ArchiveResource } from './archive/resource.js';
 export * from './archive/types.js';
 export { default as Archive } from './archive/archive.js';
 export { isArchiveCacheDisabled } from './archive/cache/is-archive-cache-disabled.js';
+export { getArchiveCacheRoot } from './archive/cache/get-archive-cache-root.js';
+export { computeArchiveCacheKey } from './archive/cache/compute-archive-cache-key.js';
+export { resolveArchiveCacheDir } from './archive/cache/resolve-archive-cache-dir.js';
+export { listArchiveCacheEntries } from './archive/cache/list-archive-cache-entries.js';
+export { clearArchiveCacheRoot } from './archive/cache/clear-archive-cache-root.js';
+export { clearArchiveCacheEntry } from './archive/cache/clear-archive-cache-entry.js';
 export { acquireArchiveLock, ArchiveLockError } from './archive/archive-lock.js';
 export { peekArchiveLockHolder } from './archive/peek-archive-lock.js';
 export { IncompatibleArchiveError } from './archive/meta/types.js';
 export { REQUIRED_FORMAT_VERSION } from './archive/meta/assert-compatible-version.js';
+export { computeBodyHash } from './archive/body-hash/compute-body-hash.js';
+export { decodeStoredBlob } from './archive/decode-html-blob.js';
+export { computeTierAAliasKey } from './archive/url-alias/compute-tier-a-alias-key.js';
+export { computeTierBAliasKey } from './archive/url-alias/compute-tier-b-alias-key.js';
 // Core
 export { DEFAULT_EXCLUDED_EXTERNAL_URLS, CrawlerOrchestrator, } from './crawler-orchestrator.js';
 export * from './types.js';
 export * from './crawler/types.js';
 export { classifyErrorKind } from './classify-error-kind.js';
+export { NETWORK_RELATED_ERROR_KINDS } from './network-related-error-kinds.js';
+export { isWithinOutageWindow } from './is-within-outage-window.js';
+export { default as NetworkOutageDetector } from './crawler/network-outage-detector.js';
+export { default as NetworkGate } from './crawler/network-gate.js';
+export { probeNetwork } from './crawler/probe-network.js';
+export { computeOutageClampTimestamp } from './archive/db-ops/outages/compute-outage-clamp-timestamp.js';
+export { chooseProbeHost } from './crawler/choose-probe-host.js';
+export { assertChromeIsInstalled } from './crawler/assert-chrome-installed.js';
 export { computeFileSha256 } from './utils/compute-file-sha256.js';
 // 0.13 ref-table population (issue #191, epic #103). Exposed as the
 // public seam that the migration script (`scripts/migrate-to-0.13.mjs`)

package/lib/is-within-outage-window.d.ts

@@ -0,0 +1,49 @@
+/**
+ * A resolved (closed) network-outage interval, as consumed by
+ * {@link isWithinOutageWindow}.
+ *
+ * This is deliberately NOT the raw `network_outages` row shape: a row whose
+ * `ended_at` is still `NULL` (the crawl session crashed before recovery was
+ * observed) must be resolved to a concrete `endedAt` by the caller — e.g.
+ * `listNetworkOutages` clamping it to the latest observed timestamp in the
+ * archive — before it reaches this function. Accepting a nullable `endedAt`
+ * here would let a crashed session's outage silently swallow every
+ * subsequent timestamp as "network-caused" forever.
+ */
+export interface OutageWindow {
+    /** Epoch ms the outage is considered to have started (already backdated to the earliest triggering error). */
+    readonly startedAt: number;
+    /** Epoch ms the outage is considered to have ended. Never `NULL` — see the interface docstring. */
+    readonly endedAt: number;
+}
+/**
+ * Decide whether a timestamp falls inside any recorded network-outage
+ * window — i.e. whether an error observed at that instant is more likely
+ * attributable to the operator's own network than to the target site.
+ *
+ * Pure and dependency-free by design: no DB handle, no `Crawler` instance.
+ * This is the single predicate every consumer (`resetFailedPages`,
+ * `listDnsBurnedHostCandidates`, `getSummary`'s attribution split) is
+ * expected to call, so the inclusive/exclusive boundary decision below is
+ * made exactly once.
+ *
+ * **Boundaries are inclusive on both ends.** A timestamp equal to
+ * `startedAt` or `endedAt` counts as inside the window. This errs toward
+ * attributing borderline errors to the outage rather than to the site:
+ * classifying a genuinely network-caused failure as "unknown/site kills it
+ * forever" (a false negative) is worse than the reverse (a false positive
+ * merely costs one extra retry pass before the kind classifier sorts it out
+ * again).
+ * @param timestamp - Epoch ms to test (typically an error's `createdAt`).
+ * @param windows - Resolved outage windows to test against. An empty array
+ *   always yields `false` — this is what makes an archive with no
+ *   `network_outages` rows (every archive created before this feature, or
+ *   any crawl with no detected outage) behave identically to today.
+ * @returns `true` if `timestamp` falls within any window.
+ * @example
+ * ```ts
+ * isWithinOutageWindow(1_000, [{ startedAt: 500, endedAt: 1_500 }]); // true
+ * isWithinOutageWindow(1_000, []); // false — no recorded outages
+ * ```
+ */
+export declare function isWithinOutageWindow(timestamp: number, windows: readonly OutageWindow[]): boolean;

package/lib/is-within-outage-window.js

@@ -0,0 +1,33 @@
+/**
+ * Decide whether a timestamp falls inside any recorded network-outage
+ * window — i.e. whether an error observed at that instant is more likely
+ * attributable to the operator's own network than to the target site.
+ *
+ * Pure and dependency-free by design: no DB handle, no `Crawler` instance.
+ * This is the single predicate every consumer (`resetFailedPages`,
+ * `listDnsBurnedHostCandidates`, `getSummary`'s attribution split) is
+ * expected to call, so the inclusive/exclusive boundary decision below is
+ * made exactly once.
+ *
+ * **Boundaries are inclusive on both ends.** A timestamp equal to
+ * `startedAt` or `endedAt` counts as inside the window. This errs toward
+ * attributing borderline errors to the outage rather than to the site:
+ * classifying a genuinely network-caused failure as "unknown/site kills it
+ * forever" (a false negative) is worse than the reverse (a false positive
+ * merely costs one extra retry pass before the kind classifier sorts it out
+ * again).
+ * @param timestamp - Epoch ms to test (typically an error's `createdAt`).
+ * @param windows - Resolved outage windows to test against. An empty array
+ *   always yields `false` — this is what makes an archive with no
+ *   `network_outages` rows (every archive created before this feature, or
+ *   any crawl with no detected outage) behave identically to today.
+ * @returns `true` if `timestamp` falls within any window.
+ * @example
+ * ```ts
+ * isWithinOutageWindow(1_000, [{ startedAt: 500, endedAt: 1_500 }]); // true
+ * isWithinOutageWindow(1_000, []); // false — no recorded outages
+ * ```
+ */
+export function isWithinOutageWindow(timestamp, windows) {
+    return windows.some((window) => timestamp >= window.startedAt && timestamp <= window.endedAt);
+}

package/lib/network-related-error-kinds.d.ts

@@ -0,0 +1,23 @@
+import type { ErrorKind } from './types.js';
+/**
+ * `ErrorKind`s that plausibly indicate trouble with the operator's own
+ * network path rather than a genuinely dead or misconfigured target site.
+ *
+ * Single source of truth shared by two consumers that must agree on
+ * exactly the same set:
+ *
+ * - `NetworkOutageDetector` — only these kinds contribute to the sliding
+ *   window that detects a suspect outage.
+ * - `evict-network-classified-destination-cache-entries.ts` — on recovery,
+ *   only `destinationCache` entries whose cached `Error` classifies into
+ *   one of these kinds are evicted (a cached `tls` or `client-blocked`
+ *   failure is a site-specific fact and must survive an outage recovery
+ *   unrelated to it).
+ *
+ * `dns` is included despite `permanent-error-kinds.ts` treating it as a
+ * permanent, site-specific verdict in isolation — that classification is
+ * exactly what outage detection exists to override when the surrounding
+ * evidence (many hosts, tight time window) points to the local network
+ * instead.
+ */
+export declare const NETWORK_RELATED_ERROR_KINDS: ReadonlySet<ErrorKind>;