@nitpicker/crawler 0.14.0 → 0.15.0

package/lib/crawler-orchestrator.d.ts

@@ -1,4 +1,5 @@
 import type { Config } from './archive/types.js';
+import type { NetworkProbe } from './crawler/probe-network.js';
 import type { InventoryMode } from './crawler/types.js';
 import type { CrawlEvent } from './types.js';
 import type { ExURL } from '@d-zero/shared/parse-url';
@@ -48,6 +49,26 @@ interface CrawlConfig extends Config {
      * this `null` so new rows are labelled `'crawled'` by the DB DEFAULT.
      */
     inventoryMode: InventoryMode | null;
+    /**
+     * See {@link CrawlerOptions.networkOutageWindowMs}. Omitted (`undefined`
+     * on the `Partial<CrawlConfig>` callers actually pass) falls through to
+     * `Crawler`'s own default — this field exists so tests can shrink the
+     * window for a fast, deterministic outage-detection cycle.
+     */
+    networkOutageWindowMs: number;
+    /** See {@link CrawlerOptions.networkOutageErrorThreshold}. */
+    networkOutageErrorThreshold: number;
+    /** See {@link CrawlerOptions.networkOutageHostThreshold}. */
+    networkOutageHostThreshold: number;
+    /** See {@link CrawlerOptions.networkOutageProbeIntervalMs}. */
+    networkOutageProbeIntervalMs: number;
+    /**
+     * See {@link CrawlerOptions.networkProbe}. The seam tests use to simulate
+     * confirmed outages and recoveries deterministically without touching
+     * the real network — plumbed through from `CrawlerOrchestrator.crawling`'s
+     * `options` so an E2E test can inject it via the public API.
+     */
+    networkProbe: NetworkProbe | null;
 }
 /**
  * Callback invoked after the CrawlerOrchestrator instance is fully initialized
@@ -56,6 +77,20 @@ interface CrawlConfig extends Config {
  * @param config - The resolved archive configuration.
  */
 type CrawlInitializedCallback = (orchestrator: CrawlerOrchestrator, config: Config) => void | Promise<void>;
+/**
+ * The CLI's already-read `--inventory` source list, passed to
+ * {@link CrawlerOrchestrator.inventory} instead of a file path — see that
+ * method's `source` param for why the path itself never crosses this
+ * boundary.
+ */
+interface InventorySource {
+    /** Lower-case hex SHA-256 digest of `bytes` (`computeFileSha256(bytes)`). */
+    sha256: string;
+    /** The exact bytes of the source list file, archived verbatim. */
+    bytes: Buffer;
+    /** Number of source-file lines the CLI warned-and-dropped for failing URL validation, before `inventoryUrls` was ever built. Recorded on the audit row as `inventory_runs.invalid_skipped`. */
+    invalidLineCount: number;
+}
 /**
  * The main entry point for Nitpicker web crawling and archiving.
  *
@@ -184,22 +219,29 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * 3. Reject archives with unfinished `pending` URLs — those would inherit
      *    the inventory `source` label by mistake. Operator must resume /
      *    retry-failed first.
-     * 4. Parse the URL list. Anything outside the archived scope is warned
-     *    and skipped (inventory is per-server by design).
-     * 5. Subtract URLs that already exist in `pages` or `resources` so the
+     * 4. If `source` is given, archive its exact bytes under
+     *    `inventory/<sha256>.txt` (see {@link Archive.saveInventorySourceList}).
+     *    Done before scope classification so even a run that discards every
+     *    URL (out of scope or already known) still leaves a recoverable
+     *    copy of what was fed in.
+     * 5. Parse the URL list — the CLI has already warned-and-dropped
+     *    unparseable-URL lines before calling this method, so every
+     *    remaining entry parses. Anything outside the archived scope is
+     *    warned and skipped (inventory is per-server by design).
+     * 6. Subtract URLs that already exist in `pages` or `resources` so the
      *    second (and N-th) inventory pass is a no-op for known rows — keeps
      *    `'inventory-seed'` rows from being silently demoted.
-     * 6. Make `<archive>.bak`. Anything thrown beyond this point restores
+     * 7. Make `<archive>.bak`. Anything thrown beyond this point restores
      *    from the backup.
-     * 7. HEAD-probe each novel URL. Responses classified as HTML are queued
+     * 8. HEAD-probe each novel URL. Responses classified as HTML are queued
      *    as Crawler seeds (`'inventory-seed'`); everything else is recorded
      *    in `resources` directly as `'inventory-seed'` (no browser launch).
-     * 8. If any HTML seeds exist, start a Crawler with
+     * 9. If any HTML seeds exist, start a Crawler with
      *    `inventoryMode = { seedUrls }` so the rendered page and every newly
      *    discovered downstream link is labelled correctly. `resume` is fed
      *    the existing `scraped` / `resources` sets so links into already-
      *    crawled pages stop at the seen-gate without re-rendering.
-     * 9. Drop the backup on success; restore it on any throw.
+     * 10. Drop the backup on success; restore it on any throw.
      *
      * Mutually exclusive with `--append` / `--retry-failed` / `--resume` /
      * `--diff` / `--list` / `--list-file` / `--single` / `--output` — the
@@ -209,20 +251,22 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * @param inventoryUrls - Pre-read URL list (one URL per element).
      * @param options - Optional config overrides — most callers leave this blank and let the archived config flow through.
      * @param initializedCallback - Hook invoked once the orchestrator is constructed but before `crawling` runs (the CLI uses it to attach progress reporting).
-     * @param sourceFileSha256 - **Pre-computed** SHA-256 hex digest of the
-     *   source URL list. The orchestrator deliberately does NOT receive
-     *   the file path: the path is privacy-sensitive (leaks user-home /
-     *   OS structure when archives are shared) and we want it lifted off
-     *   this boundary so no future log line / breadcrumb / error message
-     *   inside the orchestrator can accidentally re-leak it. The CLI
-     *   computes the digest via `computeFileSha256(resolvedListFile)`
-     *   and passes it through here. Pass `null` for programmatic
-     *   callers that built `inventoryUrls` in-memory; the audit row's
-     *   `source_file_sha256` column will be `NULL`.
+     * @param source - The CLI's already-read source list, as `{ sha256, bytes }`.
+     *   The orchestrator deliberately does NOT receive the file path: the
+     *   path is privacy-sensitive (leaks user-home / OS structure when
+     *   archives are shared) and we want it lifted off this boundary so no
+     *   future log line / breadcrumb / error message inside the orchestrator
+     *   can accidentally re-leak it. `bytes` is archived verbatim under
+     *   `inventory/<sha256>.txt` (see {@link Archive.saveInventorySourceList})
+     *   before scope classification, so a later `--inventory` run against
+     *   the same list is an audit no-op even when it discards zero new
+     *   URLs. Pass `null` for programmatic callers that built
+     *   `inventoryUrls` in-memory; the audit row's `source_file_sha256`
+     *   column will be `NULL` and no source list is archived.
      * @returns The orchestrator instance after a successful inventory pass.
      * @throws {Error} When `inventoryUrls` is empty, the archive is in list mode, or pending URLs from a previous crawl remain unresolved.
      */
-    static inventory(archivePath: string, inventoryUrls: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback, sourceFileSha256?: string | null): Promise<CrawlerOrchestrator>;
+    static inventory(archivePath: string, inventoryUrls: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback, source?: InventorySource | null): Promise<CrawlerOrchestrator>;
     /**
      * Re-fetch previously-failed pages in an existing `.nitpicker` archive.
      *

package/lib/crawler-orchestrator.js

@@ -13,6 +13,7 @@ import { dnsBurnedHostCache } from './crawler/dns-burned-host-cache.js';
 import { dnsBurnedHostShortCircuitCounter } from './crawler/dns-burned-host-short-circuit-counter.js';
 import { findScopeEntry } from './crawler/find-scope-entry.js';
 import { isLikelyHtmlUrl } from './crawler/is-likely-html-url.js';
+import { networkOutageSummaryCounter } from './crawler/network-outage-summary-counter.js';
 import { PreloadShortCircuitError } from './crawler/preload-short-circuit-error.js';
 import { protocolAgnosticKey } from './crawler/protocol-agnostic-key.js';
 import { crawlerLog, log } from './debug.js';
@@ -65,6 +66,17 @@ export class CrawlerOrchestrator extends EventEmitter {
     #crawler;
     /** Whether the crawl was started from a pre-defined URL list (non-recursive mode). */
     #fromList;
+    /**
+     * The `network_outages` row id for the currently-open outage, or `null`
+     * when none is open. Set by the `networkOutageConfirmed` handler (once
+     * the INSERT resolves) and consumed by `networkOutageRecovered` — the
+     * `Crawler` class never touches the archive itself and has no way to
+     * know the row's id, so the orchestrator is the only place that can
+     * bridge the two events for the same outage.
+     */
+    #openNetworkOutageId = null;
+    /** `startedAt` of the currently-open outage, tracked alongside {@link #openNetworkOutageId} so `networkOutageRecovered` can compute a duration for {@link networkOutageSummaryCounter}. */
+    #openNetworkOutageStartedAt = null;
     /** Serializes archive writes from crawler event handlers (FIFO). */
     #writeQueue = new WriteQueue();
     /**
@@ -138,6 +150,14 @@ export class CrawlerOrchestrator extends EventEmitter {
             // rows continue to land in pages/resources with the DB DEFAULT
             // `'crawled'` provenance label.
             inventoryMode: options?.inventoryMode ?? null,
+            // Forwarded as-is (including `undefined`) — `Crawler`'s own
+            // constructor merges each against its `DEFAULT_NETWORK_OUTAGE_*`
+            // constant, so omitting them here is exactly "use the default".
+            networkOutageWindowMs: options?.networkOutageWindowMs,
+            networkOutageErrorThreshold: options?.networkOutageErrorThreshold,
+            networkOutageHostThreshold: options?.networkOutageHostThreshold,
+            networkOutageProbeIntervalMs: options?.networkOutageProbeIntervalMs,
+            networkProbe: options?.networkProbe ?? null,
         });
     }
     /**
@@ -166,6 +186,11 @@ export class CrawlerOrchestrator extends EventEmitter {
      */
     async crawling(list, opts) {
         const writeQueue = this.#writeQueue;
+        // Per-session state, like `Crawler`'s own `#successfulHosts.clear()` /
+        // `#networkGate.open()` reset at the start of `#runDeal` — a fresh
+        // session must not inherit a dangling outage id from a prior one.
+        this.#openNetworkOutageId = null;
+        this.#openNetworkOutageStartedAt = null;
         return new Promise((resolve, reject) => {
             this.#crawler.on('error', (error) => {
                 if (error.error instanceof PreloadShortCircuitError) {
@@ -212,6 +237,67 @@ export class CrawlerOrchestrator extends EventEmitter {
                     .catch((error) => reject(error));
                 void this.emit('redirect', { result });
             });
+            this.#crawler.on('networkOutageConfirmed', ({ startedAt, detectedAt, probeHost, triggerErrorCount, triggerHostCount }) => {
+                crawlerLog('Network outage confirmed: probeHost=%s triggerErrorCount=%d triggerHostCount=%d', probeHost, triggerErrorCount, triggerHostCount);
+                // event notice; mirrors `#finalizeCrawlSession`'s unconditional
+                // `console.error` for the DNS-burn short-circuit summary.
+                console.error(`[network] outage suspected — pausing workers (probe host: ${probeHost ?? 'none'})`);
+                writeQueue
+                    .enqueue(async () => {
+                    // Both fields are set together, inside this single
+                    // closure, so the pair can never fall out of sync
+                    // (e.g. one set synchronously above while the other
+                    // waits on the INSERT) — `networkOutageRecovered`'s
+                    // queued closure always sees either both set or
+                    // neither.
+                    const id = await this.#archive.insertNetworkOutage({
+                        startedAt,
+                        detectedAt,
+                        probeHost,
+                        triggerErrorCount,
+                        triggerHostCount,
+                    });
+                    this.#openNetworkOutageId = id;
+                    this.#openNetworkOutageStartedAt = startedAt;
+                })
+                    .catch((error) => reject(error));
+            });
+            this.#crawler.on('networkOutageRecovered', ({ endedAt }) => {
+                // The `id` read is deferred to INSIDE the queued closure, not
+                // read synchronously here, because `networkOutageConfirmed`'s
+                // INSERT is itself only queued (not awaited) when that event
+                // fires — `#openNetworkOutageId` is not guaranteed to be set
+                // yet at the instant `networkOutageRecovered` fires (the two
+                // events can arrive in quick succession, e.g. in tests that
+                // drive them back-to-back with no real probe-interval delay
+                // between them). `WriteQueue` runs enqueued operations in
+                // submission order, so by the time THIS closure actually
+                // executes, the confirm's INSERT closure (enqueued first) has
+                // already completed and `#openNetworkOutageId` is reliably set.
+                writeQueue
+                    .enqueue(() => {
+                    const id = this.#openNetworkOutageId;
+                    const startedAt = this.#openNetworkOutageStartedAt;
+                    if (id === null) {
+                        // Defensive: `networkOutageConfirmed` always
+                        // precedes `networkOutageRecovered` on the same
+                        // `Crawler` instance. If this fires anyway, there
+                        // is no row to close.
+                        crawlerLog('Network outage recovered but no open outage id was tracked');
+                        return Promise.resolve();
+                    }
+                    this.#openNetworkOutageId = null;
+                    this.#openNetworkOutageStartedAt = null;
+                    const durationMs = endedAt - (startedAt ?? endedAt);
+                    networkOutageSummaryCounter.confirmedCount++;
+                    networkOutageSummaryCounter.totalDurationMs += durationMs;
+                    crawlerLog('Network outage recovered: id=%d endedAt=%d', id, endedAt);
+                    // eslint-disable-next-line no-console -- see the confirmed handler above
+                    console.error(`[network] recovered after ${Math.round(durationMs / 1000)}s`);
+                    return this.#archive.closeNetworkOutage(id, endedAt);
+                })
+                    .catch((error) => reject(error));
+            });
             this.#crawler.on('response', ({ resource, source }) => {
                 writeQueue
                     .enqueue(() => this.#archive.setResources(resource, source))
@@ -222,6 +308,11 @@ export class CrawlerOrchestrator extends EventEmitter {
                     .enqueue(() => this.#archive.setResourcesReferrers(resource))
                     .catch((error) => reject(error));
             });
+            this.#crawler.on('consoleLogs', ({ pageUrl, redirectPaths, entries }) => {
+                writeQueue
+                    .enqueue(() => this.#archive.setConsoleLogs(pageUrl, redirectPaths, entries))
+                    .catch((error) => reject(error));
+            });
             this.#crawler.on('crawlEnd', () => {
                 writeQueue
                     .drain()
@@ -387,7 +478,13 @@ export class CrawlerOrchestrator extends EventEmitter {
         const absFilePath = path.isAbsolute(archivePath)
             ? archivePath
             : path.resolve(cwd, archivePath);
-        const archive = await Archive.open({ filePath: absFilePath, cwd });
+        // See `ArchiveOpenOptions.openPluginData` for why this must be `true`
+        // on every writer path that calls `write()`.
+        const archive = await Archive.open({
+            filePath: absFilePath,
+            cwd,
+            openPluginData: true,
+        });
         // Any throw between here and the successful return must release the
         // archive lock and clean up tmpDir; the caller's `close()` only runs on
         // the happy path. Errors from `close()` itself are intentionally
@@ -481,22 +578,29 @@ export class CrawlerOrchestrator extends EventEmitter {
      * 3. Reject archives with unfinished `pending` URLs — those would inherit
      *    the inventory `source` label by mistake. Operator must resume /
      *    retry-failed first.
-     * 4. Parse the URL list. Anything outside the archived scope is warned
-     *    and skipped (inventory is per-server by design).
-     * 5. Subtract URLs that already exist in `pages` or `resources` so the
+     * 4. If `source` is given, archive its exact bytes under
+     *    `inventory/<sha256>.txt` (see {@link Archive.saveInventorySourceList}).
+     *    Done before scope classification so even a run that discards every
+     *    URL (out of scope or already known) still leaves a recoverable
+     *    copy of what was fed in.
+     * 5. Parse the URL list — the CLI has already warned-and-dropped
+     *    unparseable-URL lines before calling this method, so every
+     *    remaining entry parses. Anything outside the archived scope is
+     *    warned and skipped (inventory is per-server by design).
+     * 6. Subtract URLs that already exist in `pages` or `resources` so the
      *    second (and N-th) inventory pass is a no-op for known rows — keeps
      *    `'inventory-seed'` rows from being silently demoted.
-     * 6. Make `<archive>.bak`. Anything thrown beyond this point restores
+     * 7. Make `<archive>.bak`. Anything thrown beyond this point restores
      *    from the backup.
-     * 7. HEAD-probe each novel URL. Responses classified as HTML are queued
+     * 8. HEAD-probe each novel URL. Responses classified as HTML are queued
      *    as Crawler seeds (`'inventory-seed'`); everything else is recorded
      *    in `resources` directly as `'inventory-seed'` (no browser launch).
-     * 8. If any HTML seeds exist, start a Crawler with
+     * 9. If any HTML seeds exist, start a Crawler with
      *    `inventoryMode = { seedUrls }` so the rendered page and every newly
      *    discovered downstream link is labelled correctly. `resume` is fed
      *    the existing `scraped` / `resources` sets so links into already-
      *    crawled pages stop at the seen-gate without re-rendering.
-     * 9. Drop the backup on success; restore it on any throw.
+     * 10. Drop the backup on success; restore it on any throw.
      *
      * Mutually exclusive with `--append` / `--retry-failed` / `--resume` /
      * `--diff` / `--list` / `--list-file` / `--single` / `--output` — the
@@ -506,20 +610,22 @@ export class CrawlerOrchestrator extends EventEmitter {
      * @param inventoryUrls - Pre-read URL list (one URL per element).
      * @param options - Optional config overrides — most callers leave this blank and let the archived config flow through.
      * @param initializedCallback - Hook invoked once the orchestrator is constructed but before `crawling` runs (the CLI uses it to attach progress reporting).
-     * @param sourceFileSha256 - **Pre-computed** SHA-256 hex digest of the
-     *   source URL list. The orchestrator deliberately does NOT receive
-     *   the file path: the path is privacy-sensitive (leaks user-home /
-     *   OS structure when archives are shared) and we want it lifted off
-     *   this boundary so no future log line / breadcrumb / error message
-     *   inside the orchestrator can accidentally re-leak it. The CLI
-     *   computes the digest via `computeFileSha256(resolvedListFile)`
-     *   and passes it through here. Pass `null` for programmatic
-     *   callers that built `inventoryUrls` in-memory; the audit row's
-     *   `source_file_sha256` column will be `NULL`.
+     * @param source - The CLI's already-read source list, as `{ sha256, bytes }`.
+     *   The orchestrator deliberately does NOT receive the file path: the
+     *   path is privacy-sensitive (leaks user-home / OS structure when
+     *   archives are shared) and we want it lifted off this boundary so no
+     *   future log line / breadcrumb / error message inside the orchestrator
+     *   can accidentally re-leak it. `bytes` is archived verbatim under
+     *   `inventory/<sha256>.txt` (see {@link Archive.saveInventorySourceList})
+     *   before scope classification, so a later `--inventory` run against
+     *   the same list is an audit no-op even when it discards zero new
+     *   URLs. Pass `null` for programmatic callers that built
+     *   `inventoryUrls` in-memory; the audit row's `source_file_sha256`
+     *   column will be `NULL` and no source list is archived.
      * @returns The orchestrator instance after a successful inventory pass.
      * @throws {Error} When `inventoryUrls` is empty, the archive is in list mode, or pending URLs from a previous crawl remain unresolved.
      */
-    static async inventory(archivePath, inventoryUrls, options, initializedCallback, sourceFileSha256 = null) {
+    static async inventory(archivePath, inventoryUrls, options, initializedCallback, source = null) {
         if (inventoryUrls.length === 0) {
             throw new Error('inventory: URL list is empty');
         }
@@ -527,7 +633,13 @@ export class CrawlerOrchestrator extends EventEmitter {
         const absFilePath = path.isAbsolute(archivePath)
             ? archivePath
             : path.resolve(cwd, archivePath);
-        const archive = await Archive.open({ filePath: absFilePath, cwd });
+        // See `ArchiveOpenOptions.openPluginData` for why this must be `true`
+        // on every writer path that calls `write()`.
+        const archive = await Archive.open({
+            filePath: absFilePath,
+            cwd,
+            openPluginData: true,
+        });
         try {
             const archived = await archive.getConfig();
             if (archived.fromList) {
@@ -549,6 +661,14 @@ export class CrawlerOrchestrator extends EventEmitter {
                 // eslint-disable-next-line no-console -- operator-facing warning, must be visible regardless of DEBUG filters
                 console.warn(`inventory: archive has ${pending.length} pending URLs from a previous crawl. Proceeding — crawled-wins priority keeps their labels stable. Consider \`--resume\` first if you want the prior work finalized.`);
             }
+            // Archive the exact source bytes before scope classification, so
+            // even a run that discards every URL (out-of-scope or already
+            // known — see `novelUrls.length === 0` below) still leaves a
+            // recoverable copy of what was fed in. Content-hash-named, so a
+            // second `--inventory` pass with the same list is a no-op write.
+            if (source) {
+                await archive.saveInventorySourceList(source.sha256, source.bytes);
+            }
             // Parse + scope-classify the candidate URLs. sortUrl drops
             // unparseable strings; findScopeEntry separates in-scope from
             // out-of-scope.
@@ -703,7 +823,8 @@ export class CrawlerOrchestrator extends EventEmitter {
                     htmlSeedsCount: htmlSeeds.length,
                     nonHtmlCount: nonHtmlSeeds.length,
                     outOfScope,
-                    sourceFileSha256,
+                    sourceFileSha256: source?.sha256 ?? null,
+                    invalidSkipped: source?.invalidLineCount ?? null,
                 });
                 // Ingestion's DB writes are now committed. From here on a
                 // throw must NOT trigger the `.bak` restore (it would wipe
@@ -853,7 +974,13 @@ export class CrawlerOrchestrator extends EventEmitter {
         const absFilePath = path.isAbsolute(archivePath)
             ? archivePath
             : path.resolve(cwd, archivePath);
-        const archive = await Archive.open({ filePath: absFilePath, cwd });
+        // See `ArchiveOpenOptions.openPluginData` for why this must be `true`
+        // on every writer path that calls `write()`.
+        const archive = await Archive.open({
+            filePath: absFilePath,
+            cwd,
+            openPluginData: true,
+        });
         // Any throw between here and the successful return must release the
         // archive lock and clean up tmpDir; the caller's `close()` only runs on
         // the happy path.
@@ -990,10 +1117,10 @@ export class CrawlerOrchestrator extends EventEmitter {
      * no `--label` flag, so this is always the auto form.
      * `source_file_sha256` arrives pre-computed via
      * `aggregates.sourceFileSha256` (the CLI's `inventoryCrawl` ran
-     * `computeFileSha256` against the input txt before the orchestrator
-     * was even invoked). The orchestrator boundary deliberately never sees
-     * the absolute path — see {@link InventoryRunAggregates} for the
-     * privacy rationale.
+     * `computeFileSha256` against the bytes it read from the input txt,
+     * before the orchestrator was even invoked). The orchestrator boundary
+     * deliberately never sees the absolute path — see
+     * {@link InventoryRunAggregates} for the privacy rationale.
      *
      * **Audit-write failures abort the ingestion phase.** Swallowing them
      * would only be justified if the audit were the last write after the
@@ -1020,6 +1147,7 @@ export class CrawlerOrchestrator extends EventEmitter {
             new_pages: aggregates.htmlSeedsCount,
             new_resources: aggregates.nonHtmlCount,
             scope_skipped: aggregates.outOfScope,
+            invalid_skipped: aggregates.invalidSkipped,
         });
     }
     /**
@@ -1034,6 +1162,13 @@ export class CrawlerOrchestrator extends EventEmitter {
             // eslint-disable-next-line no-console
             console.error(`[preload] Short-circuited ${skipped} URL(s) on DNS-burned hosts`);
         }
+        const { confirmedCount, totalDurationMs } = networkOutageSummaryCounter;
+        if (confirmedCount > 0) {
+            // eslint-disable-next-line no-console
+            console.error(`[network] ${confirmedCount} outage(s), ${Math.round(totalDurationMs / 1000)}s total`);
+        }
+        networkOutageSummaryCounter.confirmedCount = 0;
+        networkOutageSummaryCounter.totalDurationMs = 0;
         clearDestinationCache();
         clearDnsBurnedHostCache();
     }

package/lib/crawler.d.ts

@@ -16,16 +16,36 @@ export { default as ArchiveResource } from './archive/resource.js';
 export * from './archive/types.js';
 export { default as Archive } from './archive/archive.js';
 export { isArchiveCacheDisabled } from './archive/cache/is-archive-cache-disabled.js';
+export { getArchiveCacheRoot } from './archive/cache/get-archive-cache-root.js';
+export { computeArchiveCacheKey } from './archive/cache/compute-archive-cache-key.js';
+export { resolveArchiveCacheDir } from './archive/cache/resolve-archive-cache-dir.js';
+export { listArchiveCacheEntries } from './archive/cache/list-archive-cache-entries.js';
+export { clearArchiveCacheRoot } from './archive/cache/clear-archive-cache-root.js';
+export { clearArchiveCacheEntry } from './archive/cache/clear-archive-cache-entry.js';
+export type { ArchiveCacheEntry, ArchiveCacheEntryKind } from './archive/cache/types.js';
 export { acquireArchiveLock, ArchiveLockError } from './archive/archive-lock.js';
 export { peekArchiveLockHolder } from './archive/peek-archive-lock.js';
 export type { ArchiveLockHolder } from './archive/peek-archive-lock.js';
 export type { FlatPageMetaColumns, JsonLdRow, JsonLdRowForInsert, TagRow, TagRowForInsert, JsonLdSummary, TagsSummary, TagInventoryEntry, PageDenormalizedColumns, } from './archive/meta/types.js';
 export { IncompatibleArchiveError } from './archive/meta/types.js';
 export { REQUIRED_FORMAT_VERSION } from './archive/meta/assert-compatible-version.js';
+export { computeBodyHash } from './archive/body-hash/compute-body-hash.js';
+export { decodeStoredBlob } from './archive/decode-html-blob.js';
+export { computeTierAAliasKey } from './archive/url-alias/compute-tier-a-alias-key.js';
+export { computeTierBAliasKey } from './archive/url-alias/compute-tier-b-alias-key.js';
 export { DEFAULT_EXCLUDED_EXTERNAL_URLS, CrawlerOrchestrator, } from './crawler-orchestrator.js';
 export * from './types.js';
 export * from './crawler/types.js';
 export { classifyErrorKind } from './classify-error-kind.js';
+export { NETWORK_RELATED_ERROR_KINDS } from './network-related-error-kinds.js';
+export type { OutageWindow } from './is-within-outage-window.js';
+export { isWithinOutageWindow } from './is-within-outage-window.js';
+export { default as NetworkOutageDetector } from './crawler/network-outage-detector.js';
+export { default as NetworkGate } from './crawler/network-gate.js';
+export type { NetworkProbe } from './crawler/probe-network.js';
+export { probeNetwork } from './crawler/probe-network.js';
+export { computeOutageClampTimestamp } from './archive/db-ops/outages/compute-outage-clamp-timestamp.js';
+export { chooseProbeHost } from './crawler/choose-probe-host.js';
 export { computeFileSha256 } from './utils/compute-file-sha256.js';
 export { populateEntityTables } from './archive/populate-entity-tables/populate-entities.js';
 export type { PageDomPathResolver } from './archive/populate-entity-tables/populate-image-items.js';
@@ -38,3 +58,4 @@ export { populateBlobRefs } from './archive/populate-ref-tables/populate-blob-re
 export { populateHeaderTables } from './archive/populate-ref-tables/populate-header-tables.js';
 export { loadResponseHeadersBySetIds } from './archive/db-ops/_shared/load-response-headers-by-set-ids.js';
 export { decodeJsonRef } from './archive/db-ops/_shared/decode-json-ref.js';
+export type { TemplateClusterBlockingEvidence, TemplateClusterBlockingReason, TemplateClusterLandmarkProfile, TemplateClusterLandmarkType, TemplateClusterReason, } from './archive/db-ops/analysis/types.js';

package/lib/crawler.js

@@ -17,15 +17,32 @@ export { default as ArchiveResource } from './archive/resource.js';
 export * from './archive/types.js';
 export { default as Archive } from './archive/archive.js';
 export { isArchiveCacheDisabled } from './archive/cache/is-archive-cache-disabled.js';
+export { getArchiveCacheRoot } from './archive/cache/get-archive-cache-root.js';
+export { computeArchiveCacheKey } from './archive/cache/compute-archive-cache-key.js';
+export { resolveArchiveCacheDir } from './archive/cache/resolve-archive-cache-dir.js';
+export { listArchiveCacheEntries } from './archive/cache/list-archive-cache-entries.js';
+export { clearArchiveCacheRoot } from './archive/cache/clear-archive-cache-root.js';
+export { clearArchiveCacheEntry } from './archive/cache/clear-archive-cache-entry.js';
 export { acquireArchiveLock, ArchiveLockError } from './archive/archive-lock.js';
 export { peekArchiveLockHolder } from './archive/peek-archive-lock.js';
 export { IncompatibleArchiveError } from './archive/meta/types.js';
 export { REQUIRED_FORMAT_VERSION } from './archive/meta/assert-compatible-version.js';
+export { computeBodyHash } from './archive/body-hash/compute-body-hash.js';
+export { decodeStoredBlob } from './archive/decode-html-blob.js';
+export { computeTierAAliasKey } from './archive/url-alias/compute-tier-a-alias-key.js';
+export { computeTierBAliasKey } from './archive/url-alias/compute-tier-b-alias-key.js';
 // Core
 export { DEFAULT_EXCLUDED_EXTERNAL_URLS, CrawlerOrchestrator, } from './crawler-orchestrator.js';
 export * from './types.js';
 export * from './crawler/types.js';
 export { classifyErrorKind } from './classify-error-kind.js';
+export { NETWORK_RELATED_ERROR_KINDS } from './network-related-error-kinds.js';
+export { isWithinOutageWindow } from './is-within-outage-window.js';
+export { default as NetworkOutageDetector } from './crawler/network-outage-detector.js';
+export { default as NetworkGate } from './crawler/network-gate.js';
+export { probeNetwork } from './crawler/probe-network.js';
+export { computeOutageClampTimestamp } from './archive/db-ops/outages/compute-outage-clamp-timestamp.js';
+export { chooseProbeHost } from './crawler/choose-probe-host.js';
 export { computeFileSha256 } from './utils/compute-file-sha256.js';
 // 0.13 ref-table population (issue #191, epic #103). Exposed as the
 // public seam that the migration script (`scripts/migrate-to-0.13.mjs`)

package/lib/is-within-outage-window.d.ts

@@ -0,0 +1,49 @@
+/**
+ * A resolved (closed) network-outage interval, as consumed by
+ * {@link isWithinOutageWindow}.
+ *
+ * This is deliberately NOT the raw `network_outages` row shape: a row whose
+ * `ended_at` is still `NULL` (the crawl session crashed before recovery was
+ * observed) must be resolved to a concrete `endedAt` by the caller — e.g.
+ * `listNetworkOutages` clamping it to the latest observed timestamp in the
+ * archive — before it reaches this function. Accepting a nullable `endedAt`
+ * here would let a crashed session's outage silently swallow every
+ * subsequent timestamp as "network-caused" forever.
+ */
+export interface OutageWindow {
+    /** Epoch ms the outage is considered to have started (already backdated to the earliest triggering error). */
+    readonly startedAt: number;
+    /** Epoch ms the outage is considered to have ended. Never `NULL` — see the interface docstring. */
+    readonly endedAt: number;
+}
+/**
+ * Decide whether a timestamp falls inside any recorded network-outage
+ * window — i.e. whether an error observed at that instant is more likely
+ * attributable to the operator's own network than to the target site.
+ *
+ * Pure and dependency-free by design: no DB handle, no `Crawler` instance.
+ * This is the single predicate every consumer (`resetFailedPages`,
+ * `listDnsBurnedHostCandidates`, `getSummary`'s attribution split) is
+ * expected to call, so the inclusive/exclusive boundary decision below is
+ * made exactly once.
+ *
+ * **Boundaries are inclusive on both ends.** A timestamp equal to
+ * `startedAt` or `endedAt` counts as inside the window. This errs toward
+ * attributing borderline errors to the outage rather than to the site:
+ * classifying a genuinely network-caused failure as "unknown/site kills it
+ * forever" (a false negative) is worse than the reverse (a false positive
+ * merely costs one extra retry pass before the kind classifier sorts it out
+ * again).
+ * @param timestamp - Epoch ms to test (typically an error's `createdAt`).
+ * @param windows - Resolved outage windows to test against. An empty array
+ *   always yields `false` — this is what makes an archive with no
+ *   `network_outages` rows (every archive created before this feature, or
+ *   any crawl with no detected outage) behave identically to today.
+ * @returns `true` if `timestamp` falls within any window.
+ * @example
+ * ```ts
+ * isWithinOutageWindow(1_000, [{ startedAt: 500, endedAt: 1_500 }]); // true
+ * isWithinOutageWindow(1_000, []); // false — no recorded outages
+ * ```
+ */
+export declare function isWithinOutageWindow(timestamp: number, windows: readonly OutageWindow[]): boolean;

package/lib/is-within-outage-window.js

@@ -0,0 +1,33 @@
+/**
+ * Decide whether a timestamp falls inside any recorded network-outage
+ * window — i.e. whether an error observed at that instant is more likely
+ * attributable to the operator's own network than to the target site.
+ *
+ * Pure and dependency-free by design: no DB handle, no `Crawler` instance.
+ * This is the single predicate every consumer (`resetFailedPages`,
+ * `listDnsBurnedHostCandidates`, `getSummary`'s attribution split) is
+ * expected to call, so the inclusive/exclusive boundary decision below is
+ * made exactly once.
+ *
+ * **Boundaries are inclusive on both ends.** A timestamp equal to
+ * `startedAt` or `endedAt` counts as inside the window. This errs toward
+ * attributing borderline errors to the outage rather than to the site:
+ * classifying a genuinely network-caused failure as "unknown/site kills it
+ * forever" (a false negative) is worse than the reverse (a false positive
+ * merely costs one extra retry pass before the kind classifier sorts it out
+ * again).
+ * @param timestamp - Epoch ms to test (typically an error's `createdAt`).
+ * @param windows - Resolved outage windows to test against. An empty array
+ *   always yields `false` — this is what makes an archive with no
+ *   `network_outages` rows (every archive created before this feature, or
+ *   any crawl with no detected outage) behave identically to today.
+ * @returns `true` if `timestamp` falls within any window.
+ * @example
+ * ```ts
+ * isWithinOutageWindow(1_000, [{ startedAt: 500, endedAt: 1_500 }]); // true
+ * isWithinOutageWindow(1_000, []); // false — no recorded outages
+ * ```
+ */
+export function isWithinOutageWindow(timestamp, windows) {
+    return windows.some((window) => timestamp >= window.startedAt && timestamp <= window.endedAt);
+}

package/lib/network-related-error-kinds.d.ts

@@ -0,0 +1,23 @@
+import type { ErrorKind } from './types.js';
+/**
+ * `ErrorKind`s that plausibly indicate trouble with the operator's own
+ * network path rather than a genuinely dead or misconfigured target site.
+ *
+ * Single source of truth shared by two consumers that must agree on
+ * exactly the same set:
+ *
+ * - `NetworkOutageDetector` — only these kinds contribute to the sliding
+ *   window that detects a suspect outage.
+ * - `evict-network-classified-destination-cache-entries.ts` — on recovery,
+ *   only `destinationCache` entries whose cached `Error` classifies into
+ *   one of these kinds are evicted (a cached `tls` or `client-blocked`
+ *   failure is a site-specific fact and must survive an outage recovery
+ *   unrelated to it).
+ *
+ * `dns` is included despite `permanent-error-kinds.ts` treating it as a
+ * permanent, site-specific verdict in isolation — that classification is
+ * exactly what outage detection exists to override when the surrounding
+ * evidence (many hosts, tight time window) points to the local network
+ * instead.
+ */
+export declare const NETWORK_RELATED_ERROR_KINDS: ReadonlySet<ErrorKind>;