@nitpicker/crawler 0.13.0 → 0.15.0

package/lib/crawler-orchestrator.js

@@ -13,6 +13,7 @@ import { dnsBurnedHostCache } from './crawler/dns-burned-host-cache.js';
 import { dnsBurnedHostShortCircuitCounter } from './crawler/dns-burned-host-short-circuit-counter.js';
 import { findScopeEntry } from './crawler/find-scope-entry.js';
 import { isLikelyHtmlUrl } from './crawler/is-likely-html-url.js';
+import { networkOutageSummaryCounter } from './crawler/network-outage-summary-counter.js';
 import { PreloadShortCircuitError } from './crawler/preload-short-circuit-error.js';
 import { protocolAgnosticKey } from './crawler/protocol-agnostic-key.js';
 import { crawlerLog, log } from './debug.js';
@@ -65,6 +66,17 @@ export class CrawlerOrchestrator extends EventEmitter {
     #crawler;
     /** Whether the crawl was started from a pre-defined URL list (non-recursive mode). */
     #fromList;
+    /**
+     * The `network_outages` row id for the currently-open outage, or `null`
+     * when none is open. Set by the `networkOutageConfirmed` handler (once
+     * the INSERT resolves) and consumed by `networkOutageRecovered` — the
+     * `Crawler` class never touches the archive itself and has no way to
+     * know the row's id, so the orchestrator is the only place that can
+     * bridge the two events for the same outage.
+     */
+    #openNetworkOutageId = null;
+    /** `startedAt` of the currently-open outage, tracked alongside {@link #openNetworkOutageId} so `networkOutageRecovered` can compute a duration for {@link networkOutageSummaryCounter}. */
+    #openNetworkOutageStartedAt = null;
     /** Serializes archive writes from crawler event handlers (FIFO). */
     #writeQueue = new WriteQueue();
     /**
@@ -109,6 +121,7 @@ export class CrawlerOrchestrator extends EventEmitter {
             verbose: options?.verbose ?? false,
             userAgent: options?.userAgent || defaultUserAgent,
             ignoreRobots: options?.ignoreRobots ?? false,
+            mainContentSelector: options?.mainContentSelector ?? null,
             // Let the crawler reuse sub-resource data captured during page
             // rendering instead of issuing a redundant HEAD pre-flight.
             lookupResource: async (urls) => {
@@ -137,6 +150,14 @@ export class CrawlerOrchestrator extends EventEmitter {
             // rows continue to land in pages/resources with the DB DEFAULT
             // `'crawled'` provenance label.
             inventoryMode: options?.inventoryMode ?? null,
+            // Forwarded as-is (including `undefined`) — `Crawler`'s own
+            // constructor merges each against its `DEFAULT_NETWORK_OUTAGE_*`
+            // constant, so omitting them here is exactly "use the default".
+            networkOutageWindowMs: options?.networkOutageWindowMs,
+            networkOutageErrorThreshold: options?.networkOutageErrorThreshold,
+            networkOutageHostThreshold: options?.networkOutageHostThreshold,
+            networkOutageProbeIntervalMs: options?.networkOutageProbeIntervalMs,
+            networkProbe: options?.networkProbe ?? null,
         });
     }
     /**
@@ -165,6 +186,11 @@ export class CrawlerOrchestrator extends EventEmitter {
      */
     async crawling(list, opts) {
         const writeQueue = this.#writeQueue;
+        // Per-session state, like `Crawler`'s own `#successfulHosts.clear()` /
+        // `#networkGate.open()` reset at the start of `#runDeal` — a fresh
+        // session must not inherit a dangling outage id from a prior one.
+        this.#openNetworkOutageId = null;
+        this.#openNetworkOutageStartedAt = null;
         return new Promise((resolve, reject) => {
             this.#crawler.on('error', (error) => {
                 if (error.error instanceof PreloadShortCircuitError) {
@@ -211,6 +237,67 @@ export class CrawlerOrchestrator extends EventEmitter {
                     .catch((error) => reject(error));
                 void this.emit('redirect', { result });
             });
+            this.#crawler.on('networkOutageConfirmed', ({ startedAt, detectedAt, probeHost, triggerErrorCount, triggerHostCount }) => {
+                crawlerLog('Network outage confirmed: probeHost=%s triggerErrorCount=%d triggerHostCount=%d', probeHost, triggerErrorCount, triggerHostCount);
+                // event notice; mirrors `#finalizeCrawlSession`'s unconditional
+                // `console.error` for the DNS-burn short-circuit summary.
+                console.error(`[network] outage suspected — pausing workers (probe host: ${probeHost ?? 'none'})`);
+                writeQueue
+                    .enqueue(async () => {
+                    // Both fields are set together, inside this single
+                    // closure, so the pair can never fall out of sync
+                    // (e.g. one set synchronously above while the other
+                    // waits on the INSERT) — `networkOutageRecovered`'s
+                    // queued closure always sees either both set or
+                    // neither.
+                    const id = await this.#archive.insertNetworkOutage({
+                        startedAt,
+                        detectedAt,
+                        probeHost,
+                        triggerErrorCount,
+                        triggerHostCount,
+                    });
+                    this.#openNetworkOutageId = id;
+                    this.#openNetworkOutageStartedAt = startedAt;
+                })
+                    .catch((error) => reject(error));
+            });
+            this.#crawler.on('networkOutageRecovered', ({ endedAt }) => {
+                // The `id` read is deferred to INSIDE the queued closure, not
+                // read synchronously here, because `networkOutageConfirmed`'s
+                // INSERT is itself only queued (not awaited) when that event
+                // fires — `#openNetworkOutageId` is not guaranteed to be set
+                // yet at the instant `networkOutageRecovered` fires (the two
+                // events can arrive in quick succession, e.g. in tests that
+                // drive them back-to-back with no real probe-interval delay
+                // between them). `WriteQueue` runs enqueued operations in
+                // submission order, so by the time THIS closure actually
+                // executes, the confirm's INSERT closure (enqueued first) has
+                // already completed and `#openNetworkOutageId` is reliably set.
+                writeQueue
+                    .enqueue(() => {
+                    const id = this.#openNetworkOutageId;
+                    const startedAt = this.#openNetworkOutageStartedAt;
+                    if (id === null) {
+                        // Defensive: `networkOutageConfirmed` always
+                        // precedes `networkOutageRecovered` on the same
+                        // `Crawler` instance. If this fires anyway, there
+                        // is no row to close.
+                        crawlerLog('Network outage recovered but no open outage id was tracked');
+                        return Promise.resolve();
+                    }
+                    this.#openNetworkOutageId = null;
+                    this.#openNetworkOutageStartedAt = null;
+                    const durationMs = endedAt - (startedAt ?? endedAt);
+                    networkOutageSummaryCounter.confirmedCount++;
+                    networkOutageSummaryCounter.totalDurationMs += durationMs;
+                    crawlerLog('Network outage recovered: id=%d endedAt=%d', id, endedAt);
+                    // eslint-disable-next-line no-console -- see the confirmed handler above
+                    console.error(`[network] recovered after ${Math.round(durationMs / 1000)}s`);
+                    return this.#archive.closeNetworkOutage(id, endedAt);
+                })
+                    .catch((error) => reject(error));
+            });
             this.#crawler.on('response', ({ resource, source }) => {
                 writeQueue
                     .enqueue(() => this.#archive.setResources(resource, source))
@@ -221,6 +308,11 @@ export class CrawlerOrchestrator extends EventEmitter {
                     .enqueue(() => this.#archive.setResourcesReferrers(resource))
                     .catch((error) => reject(error));
             });
+            this.#crawler.on('consoleLogs', ({ pageUrl, redirectPaths, entries }) => {
+                writeQueue
+                    .enqueue(() => this.#archive.setConsoleLogs(pageUrl, redirectPaths, entries))
+                    .catch((error) => reject(error));
+            });
             this.#crawler.on('crawlEnd', () => {
                 writeQueue
                     .drain()
@@ -337,6 +429,7 @@ export class CrawlerOrchestrator extends EventEmitter {
             disableQueries,
             userAgent: options?.userAgent || defaultUserAgent,
             ignoreRobots: options?.ignoreRobots ?? false,
+            mainContentSelector: options?.mainContentSelector ?? null,
         });
         const orchestrator = new CrawlerOrchestrator(archive, {
             ...options,
@@ -385,7 +478,13 @@ export class CrawlerOrchestrator extends EventEmitter {
         const absFilePath = path.isAbsolute(archivePath)
             ? archivePath
             : path.resolve(cwd, archivePath);
-        const archive = await Archive.open({ filePath: absFilePath, cwd });
+        // See `ArchiveOpenOptions.openPluginData` for why this must be `true`
+        // on every writer path that calls `write()`.
+        const archive = await Archive.open({
+            filePath: absFilePath,
+            cwd,
+            openPluginData: true,
+        });
         // Any throw between here and the successful return must release the
         // archive lock and clean up tmpDir; the caller's `close()` only runs on
         // the happy path. Errors from `close()` itself are intentionally
@@ -479,22 +578,29 @@ export class CrawlerOrchestrator extends EventEmitter {
      * 3. Reject archives with unfinished `pending` URLs — those would inherit
      *    the inventory `source` label by mistake. Operator must resume /
      *    retry-failed first.
-     * 4. Parse the URL list. Anything outside the archived scope is warned
-     *    and skipped (inventory is per-server by design).
-     * 5. Subtract URLs that already exist in `pages` or `resources` so the
+     * 4. If `source` is given, archive its exact bytes under
+     *    `inventory/<sha256>.txt` (see {@link Archive.saveInventorySourceList}).
+     *    Done before scope classification so even a run that discards every
+     *    URL (out of scope or already known) still leaves a recoverable
+     *    copy of what was fed in.
+     * 5. Parse the URL list — the CLI has already warned-and-dropped
+     *    unparseable-URL lines before calling this method, so every
+     *    remaining entry parses. Anything outside the archived scope is
+     *    warned and skipped (inventory is per-server by design).
+     * 6. Subtract URLs that already exist in `pages` or `resources` so the
      *    second (and N-th) inventory pass is a no-op for known rows — keeps
      *    `'inventory-seed'` rows from being silently demoted.
-     * 6. Make `<archive>.bak`. Anything thrown beyond this point restores
+     * 7. Make `<archive>.bak`. Anything thrown beyond this point restores
      *    from the backup.
-     * 7. HEAD-probe each novel URL. Responses classified as HTML are queued
+     * 8. HEAD-probe each novel URL. Responses classified as HTML are queued
      *    as Crawler seeds (`'inventory-seed'`); everything else is recorded
      *    in `resources` directly as `'inventory-seed'` (no browser launch).
-     * 8. If any HTML seeds exist, start a Crawler with
+     * 9. If any HTML seeds exist, start a Crawler with
      *    `inventoryMode = { seedUrls }` so the rendered page and every newly
      *    discovered downstream link is labelled correctly. `resume` is fed
      *    the existing `scraped` / `resources` sets so links into already-
      *    crawled pages stop at the seen-gate without re-rendering.
-     * 9. Drop the backup on success; restore it on any throw.
+     * 10. Drop the backup on success; restore it on any throw.
      *
      * Mutually exclusive with `--append` / `--retry-failed` / `--resume` /
      * `--diff` / `--list` / `--list-file` / `--single` / `--output` — the
@@ -504,20 +610,22 @@ export class CrawlerOrchestrator extends EventEmitter {
      * @param inventoryUrls - Pre-read URL list (one URL per element).
      * @param options - Optional config overrides — most callers leave this blank and let the archived config flow through.
      * @param initializedCallback - Hook invoked once the orchestrator is constructed but before `crawling` runs (the CLI uses it to attach progress reporting).
-     * @param sourceFileSha256 - **Pre-computed** SHA-256 hex digest of the
-     *   source URL list. The orchestrator deliberately does NOT receive
-     *   the file path: the path is privacy-sensitive (leaks user-home /
-     *   OS structure when archives are shared) and we want it lifted off
-     *   this boundary so no future log line / breadcrumb / error message
-     *   inside the orchestrator can accidentally re-leak it. The CLI
-     *   computes the digest via `computeFileSha256(resolvedListFile)`
-     *   and passes it through here. Pass `null` for programmatic
-     *   callers that built `inventoryUrls` in-memory; the audit row's
-     *   `source_file_sha256` column will be `NULL`.
+     * @param source - The CLI's already-read source list, as `{ sha256, bytes }`.
+     *   The orchestrator deliberately does NOT receive the file path: the
+     *   path is privacy-sensitive (leaks user-home / OS structure when
+     *   archives are shared) and we want it lifted off this boundary so no
+     *   future log line / breadcrumb / error message inside the orchestrator
+     *   can accidentally re-leak it. `bytes` is archived verbatim under
+     *   `inventory/<sha256>.txt` (see {@link Archive.saveInventorySourceList})
+     *   before scope classification, so a later `--inventory` run against
+     *   the same list is an audit no-op even when it discards zero new
+     *   URLs. Pass `null` for programmatic callers that built
+     *   `inventoryUrls` in-memory; the audit row's `source_file_sha256`
+     *   column will be `NULL` and no source list is archived.
      * @returns The orchestrator instance after a successful inventory pass.
      * @throws {Error} When `inventoryUrls` is empty, the archive is in list mode, or pending URLs from a previous crawl remain unresolved.
      */
-    static async inventory(archivePath, inventoryUrls, options, initializedCallback, sourceFileSha256 = null) {
+    static async inventory(archivePath, inventoryUrls, options, initializedCallback, source = null) {
         if (inventoryUrls.length === 0) {
             throw new Error('inventory: URL list is empty');
         }
@@ -525,7 +633,13 @@ export class CrawlerOrchestrator extends EventEmitter {
         const absFilePath = path.isAbsolute(archivePath)
             ? archivePath
             : path.resolve(cwd, archivePath);
-        const archive = await Archive.open({ filePath: absFilePath, cwd });
+        // See `ArchiveOpenOptions.openPluginData` for why this must be `true`
+        // on every writer path that calls `write()`.
+        const archive = await Archive.open({
+            filePath: absFilePath,
+            cwd,
+            openPluginData: true,
+        });
         try {
             const archived = await archive.getConfig();
             if (archived.fromList) {
@@ -547,6 +661,14 @@ export class CrawlerOrchestrator extends EventEmitter {
                 // eslint-disable-next-line no-console -- operator-facing warning, must be visible regardless of DEBUG filters
                 console.warn(`inventory: archive has ${pending.length} pending URLs from a previous crawl. Proceeding — crawled-wins priority keeps their labels stable. Consider \`--resume\` first if you want the prior work finalized.`);
             }
+            // Archive the exact source bytes before scope classification, so
+            // even a run that discards every URL (out-of-scope or already
+            // known — see `novelUrls.length === 0` below) still leaves a
+            // recoverable copy of what was fed in. Content-hash-named, so a
+            // second `--inventory` pass with the same list is a no-op write.
+            if (source) {
+                await archive.saveInventorySourceList(source.sha256, source.bytes);
+            }
             // Parse + scope-classify the candidate URLs. sortUrl drops
             // unparseable strings; findScopeEntry separates in-scope from
             // out-of-scope.
@@ -701,7 +823,8 @@ export class CrawlerOrchestrator extends EventEmitter {
                     htmlSeedsCount: htmlSeeds.length,
                     nonHtmlCount: nonHtmlSeeds.length,
                     outOfScope,
-                    sourceFileSha256,
+                    sourceFileSha256: source?.sha256 ?? null,
+                    invalidSkipped: source?.invalidLineCount ?? null,
                 });
                 // Ingestion's DB writes are now committed. From here on a
                 // throw must NOT trigger the `.bak` restore (it would wipe
@@ -851,7 +974,13 @@ export class CrawlerOrchestrator extends EventEmitter {
         const absFilePath = path.isAbsolute(archivePath)
             ? archivePath
             : path.resolve(cwd, archivePath);
-        const archive = await Archive.open({ filePath: absFilePath, cwd });
+        // See `ArchiveOpenOptions.openPluginData` for why this must be `true`
+        // on every writer path that calls `write()`.
+        const archive = await Archive.open({
+            filePath: absFilePath,
+            cwd,
+            openPluginData: true,
+        });
         // Any throw between here and the successful return must release the
         // archive lock and clean up tmpDir; the caller's `close()` only runs on
         // the happy path.
@@ -988,10 +1117,10 @@ export class CrawlerOrchestrator extends EventEmitter {
      * no `--label` flag, so this is always the auto form.
      * `source_file_sha256` arrives pre-computed via
      * `aggregates.sourceFileSha256` (the CLI's `inventoryCrawl` ran
-     * `computeFileSha256` against the input txt before the orchestrator
-     * was even invoked). The orchestrator boundary deliberately never sees
-     * the absolute path — see {@link InventoryRunAggregates} for the
-     * privacy rationale.
+     * `computeFileSha256` against the bytes it read from the input txt,
+     * before the orchestrator was even invoked). The orchestrator boundary
+     * deliberately never sees the absolute path — see
+     * {@link InventoryRunAggregates} for the privacy rationale.
      *
      * **Audit-write failures abort the ingestion phase.** Swallowing them
      * would only be justified if the audit were the last write after the
@@ -1018,6 +1147,7 @@ export class CrawlerOrchestrator extends EventEmitter {
             new_pages: aggregates.htmlSeedsCount,
             new_resources: aggregates.nonHtmlCount,
             scope_skipped: aggregates.outOfScope,
+            invalid_skipped: aggregates.invalidSkipped,
         });
     }
     /**
@@ -1032,6 +1162,13 @@ export class CrawlerOrchestrator extends EventEmitter {
             // eslint-disable-next-line no-console
             console.error(`[preload] Short-circuited ${skipped} URL(s) on DNS-burned hosts`);
         }
+        const { confirmedCount, totalDurationMs } = networkOutageSummaryCounter;
+        if (confirmedCount > 0) {
+            // eslint-disable-next-line no-console
+            console.error(`[network] ${confirmedCount} outage(s), ${Math.round(totalDurationMs / 1000)}s total`);
+        }
+        networkOutageSummaryCounter.confirmedCount = 0;
+        networkOutageSummaryCounter.totalDurationMs = 0;
         clearDestinationCache();
         clearDnsBurnedHostCache();
     }

package/lib/crawler.d.ts

@@ -16,16 +16,36 @@ export { default as ArchiveResource } from './archive/resource.js';
 export * from './archive/types.js';
 export { default as Archive } from './archive/archive.js';
 export { isArchiveCacheDisabled } from './archive/cache/is-archive-cache-disabled.js';
+export { getArchiveCacheRoot } from './archive/cache/get-archive-cache-root.js';
+export { computeArchiveCacheKey } from './archive/cache/compute-archive-cache-key.js';
+export { resolveArchiveCacheDir } from './archive/cache/resolve-archive-cache-dir.js';
+export { listArchiveCacheEntries } from './archive/cache/list-archive-cache-entries.js';
+export { clearArchiveCacheRoot } from './archive/cache/clear-archive-cache-root.js';
+export { clearArchiveCacheEntry } from './archive/cache/clear-archive-cache-entry.js';
+export type { ArchiveCacheEntry, ArchiveCacheEntryKind } from './archive/cache/types.js';
 export { acquireArchiveLock, ArchiveLockError } from './archive/archive-lock.js';
 export { peekArchiveLockHolder } from './archive/peek-archive-lock.js';
 export type { ArchiveLockHolder } from './archive/peek-archive-lock.js';
 export type { FlatPageMetaColumns, JsonLdRow, JsonLdRowForInsert, TagRow, TagRowForInsert, JsonLdSummary, TagsSummary, TagInventoryEntry, PageDenormalizedColumns, } from './archive/meta/types.js';
 export { IncompatibleArchiveError } from './archive/meta/types.js';
 export { REQUIRED_FORMAT_VERSION } from './archive/meta/assert-compatible-version.js';
+export { computeBodyHash } from './archive/body-hash/compute-body-hash.js';
+export { decodeStoredBlob } from './archive/decode-html-blob.js';
+export { computeTierAAliasKey } from './archive/url-alias/compute-tier-a-alias-key.js';
+export { computeTierBAliasKey } from './archive/url-alias/compute-tier-b-alias-key.js';
 export { DEFAULT_EXCLUDED_EXTERNAL_URLS, CrawlerOrchestrator, } from './crawler-orchestrator.js';
 export * from './types.js';
 export * from './crawler/types.js';
 export { classifyErrorKind } from './classify-error-kind.js';
+export { NETWORK_RELATED_ERROR_KINDS } from './network-related-error-kinds.js';
+export type { OutageWindow } from './is-within-outage-window.js';
+export { isWithinOutageWindow } from './is-within-outage-window.js';
+export { default as NetworkOutageDetector } from './crawler/network-outage-detector.js';
+export { default as NetworkGate } from './crawler/network-gate.js';
+export type { NetworkProbe } from './crawler/probe-network.js';
+export { probeNetwork } from './crawler/probe-network.js';
+export { computeOutageClampTimestamp } from './archive/db-ops/outages/compute-outage-clamp-timestamp.js';
+export { chooseProbeHost } from './crawler/choose-probe-host.js';
 export { computeFileSha256 } from './utils/compute-file-sha256.js';
 export { populateEntityTables } from './archive/populate-entity-tables/populate-entities.js';
 export type { PageDomPathResolver } from './archive/populate-entity-tables/populate-image-items.js';
@@ -38,3 +58,4 @@ export { populateBlobRefs } from './archive/populate-ref-tables/populate-blob-re
 export { populateHeaderTables } from './archive/populate-ref-tables/populate-header-tables.js';
 export { loadResponseHeadersBySetIds } from './archive/db-ops/_shared/load-response-headers-by-set-ids.js';
 export { decodeJsonRef } from './archive/db-ops/_shared/decode-json-ref.js';
+export type { TemplateClusterBlockingEvidence, TemplateClusterBlockingReason, TemplateClusterLandmarkProfile, TemplateClusterLandmarkType, TemplateClusterReason, } from './archive/db-ops/analysis/types.js';

package/lib/crawler.js

@@ -17,15 +17,32 @@ export { default as ArchiveResource } from './archive/resource.js';
 export * from './archive/types.js';
 export { default as Archive } from './archive/archive.js';
 export { isArchiveCacheDisabled } from './archive/cache/is-archive-cache-disabled.js';
+export { getArchiveCacheRoot } from './archive/cache/get-archive-cache-root.js';
+export { computeArchiveCacheKey } from './archive/cache/compute-archive-cache-key.js';
+export { resolveArchiveCacheDir } from './archive/cache/resolve-archive-cache-dir.js';
+export { listArchiveCacheEntries } from './archive/cache/list-archive-cache-entries.js';
+export { clearArchiveCacheRoot } from './archive/cache/clear-archive-cache-root.js';
+export { clearArchiveCacheEntry } from './archive/cache/clear-archive-cache-entry.js';
 export { acquireArchiveLock, ArchiveLockError } from './archive/archive-lock.js';
 export { peekArchiveLockHolder } from './archive/peek-archive-lock.js';
 export { IncompatibleArchiveError } from './archive/meta/types.js';
 export { REQUIRED_FORMAT_VERSION } from './archive/meta/assert-compatible-version.js';
+export { computeBodyHash } from './archive/body-hash/compute-body-hash.js';
+export { decodeStoredBlob } from './archive/decode-html-blob.js';
+export { computeTierAAliasKey } from './archive/url-alias/compute-tier-a-alias-key.js';
+export { computeTierBAliasKey } from './archive/url-alias/compute-tier-b-alias-key.js';
 // Core
 export { DEFAULT_EXCLUDED_EXTERNAL_URLS, CrawlerOrchestrator, } from './crawler-orchestrator.js';
 export * from './types.js';
 export * from './crawler/types.js';
 export { classifyErrorKind } from './classify-error-kind.js';
+export { NETWORK_RELATED_ERROR_KINDS } from './network-related-error-kinds.js';
+export { isWithinOutageWindow } from './is-within-outage-window.js';
+export { default as NetworkOutageDetector } from './crawler/network-outage-detector.js';
+export { default as NetworkGate } from './crawler/network-gate.js';
+export { probeNetwork } from './crawler/probe-network.js';
+export { computeOutageClampTimestamp } from './archive/db-ops/outages/compute-outage-clamp-timestamp.js';
+export { chooseProbeHost } from './crawler/choose-probe-host.js';
 export { computeFileSha256 } from './utils/compute-file-sha256.js';
 // 0.13 ref-table population (issue #191, epic #103). Exposed as the
 // public seam that the migration script (`scripts/migrate-to-0.13.mjs`)

package/lib/is-within-outage-window.d.ts

@@ -0,0 +1,49 @@
+/**
+ * A resolved (closed) network-outage interval, as consumed by
+ * {@link isWithinOutageWindow}.
+ *
+ * This is deliberately NOT the raw `network_outages` row shape: a row whose
+ * `ended_at` is still `NULL` (the crawl session crashed before recovery was
+ * observed) must be resolved to a concrete `endedAt` by the caller — e.g.
+ * `listNetworkOutages` clamping it to the latest observed timestamp in the
+ * archive — before it reaches this function. Accepting a nullable `endedAt`
+ * here would let a crashed session's outage silently swallow every
+ * subsequent timestamp as "network-caused" forever.
+ */
+export interface OutageWindow {
+    /** Epoch ms the outage is considered to have started (already backdated to the earliest triggering error). */
+    readonly startedAt: number;
+    /** Epoch ms the outage is considered to have ended. Never `NULL` — see the interface docstring. */
+    readonly endedAt: number;
+}
+/**
+ * Decide whether a timestamp falls inside any recorded network-outage
+ * window — i.e. whether an error observed at that instant is more likely
+ * attributable to the operator's own network than to the target site.
+ *
+ * Pure and dependency-free by design: no DB handle, no `Crawler` instance.
+ * This is the single predicate every consumer (`resetFailedPages`,
+ * `listDnsBurnedHostCandidates`, `getSummary`'s attribution split) is
+ * expected to call, so the inclusive/exclusive boundary decision below is
+ * made exactly once.
+ *
+ * **Boundaries are inclusive on both ends.** A timestamp equal to
+ * `startedAt` or `endedAt` counts as inside the window. This errs toward
+ * attributing borderline errors to the outage rather than to the site:
+ * classifying a genuinely network-caused failure as "unknown/site kills it
+ * forever" (a false negative) is worse than the reverse (a false positive
+ * merely costs one extra retry pass before the kind classifier sorts it out
+ * again).
+ * @param timestamp - Epoch ms to test (typically an error's `createdAt`).
+ * @param windows - Resolved outage windows to test against. An empty array
+ *   always yields `false` — this is what makes an archive with no
+ *   `network_outages` rows (every archive created before this feature, or
+ *   any crawl with no detected outage) behave identically to today.
+ * @returns `true` if `timestamp` falls within any window.
+ * @example
+ * ```ts
+ * isWithinOutageWindow(1_000, [{ startedAt: 500, endedAt: 1_500 }]); // true
+ * isWithinOutageWindow(1_000, []); // false — no recorded outages
+ * ```
+ */
+export declare function isWithinOutageWindow(timestamp: number, windows: readonly OutageWindow[]): boolean;

package/lib/is-within-outage-window.js

@@ -0,0 +1,33 @@
+/**
+ * Decide whether a timestamp falls inside any recorded network-outage
+ * window — i.e. whether an error observed at that instant is more likely
+ * attributable to the operator's own network than to the target site.
+ *
+ * Pure and dependency-free by design: no DB handle, no `Crawler` instance.
+ * This is the single predicate every consumer (`resetFailedPages`,
+ * `listDnsBurnedHostCandidates`, `getSummary`'s attribution split) is
+ * expected to call, so the inclusive/exclusive boundary decision below is
+ * made exactly once.
+ *
+ * **Boundaries are inclusive on both ends.** A timestamp equal to
+ * `startedAt` or `endedAt` counts as inside the window. This errs toward
+ * attributing borderline errors to the outage rather than to the site:
+ * classifying a genuinely network-caused failure as "unknown/site kills it
+ * forever" (a false negative) is worse than the reverse (a false positive
+ * merely costs one extra retry pass before the kind classifier sorts it out
+ * again).
+ * @param timestamp - Epoch ms to test (typically an error's `createdAt`).
+ * @param windows - Resolved outage windows to test against. An empty array
+ *   always yields `false` — this is what makes an archive with no
+ *   `network_outages` rows (every archive created before this feature, or
+ *   any crawl with no detected outage) behave identically to today.
+ * @returns `true` if `timestamp` falls within any window.
+ * @example
+ * ```ts
+ * isWithinOutageWindow(1_000, [{ startedAt: 500, endedAt: 1_500 }]); // true
+ * isWithinOutageWindow(1_000, []); // false — no recorded outages
+ * ```
+ */
+export function isWithinOutageWindow(timestamp, windows) {
+    return windows.some((window) => timestamp >= window.startedAt && timestamp <= window.endedAt);
+}

package/lib/network-related-error-kinds.d.ts

@@ -0,0 +1,23 @@
+import type { ErrorKind } from './types.js';
+/**
+ * `ErrorKind`s that plausibly indicate trouble with the operator's own
+ * network path rather than a genuinely dead or misconfigured target site.
+ *
+ * Single source of truth shared by two consumers that must agree on
+ * exactly the same set:
+ *
+ * - `NetworkOutageDetector` — only these kinds contribute to the sliding
+ *   window that detects a suspect outage.
+ * - `evict-network-classified-destination-cache-entries.ts` — on recovery,
+ *   only `destinationCache` entries whose cached `Error` classifies into
+ *   one of these kinds are evicted (a cached `tls` or `client-blocked`
+ *   failure is a site-specific fact and must survive an outage recovery
+ *   unrelated to it).
+ *
+ * `dns` is included despite `permanent-error-kinds.ts` treating it as a
+ * permanent, site-specific verdict in isolation — that classification is
+ * exactly what outage detection exists to override when the surrounding
+ * evidence (many hosts, tight time window) points to the local network
+ * instead.
+ */
+export declare const NETWORK_RELATED_ERROR_KINDS: ReadonlySet<ErrorKind>;

package/lib/network-related-error-kinds.js

@@ -0,0 +1,28 @@
+/**
+ * `ErrorKind`s that plausibly indicate trouble with the operator's own
+ * network path rather than a genuinely dead or misconfigured target site.
+ *
+ * Single source of truth shared by two consumers that must agree on
+ * exactly the same set:
+ *
+ * - `NetworkOutageDetector` — only these kinds contribute to the sliding
+ *   window that detects a suspect outage.
+ * - `evict-network-classified-destination-cache-entries.ts` — on recovery,
+ *   only `destinationCache` entries whose cached `Error` classifies into
+ *   one of these kinds are evicted (a cached `tls` or `client-blocked`
+ *   failure is a site-specific fact and must survive an outage recovery
+ *   unrelated to it).
+ *
+ * `dns` is included despite `permanent-error-kinds.ts` treating it as a
+ * permanent, site-specific verdict in isolation — that classification is
+ * exactly what outage detection exists to override when the surrounding
+ * evidence (many hosts, tight time window) points to the local network
+ * instead.
+ */
+export const NETWORK_RELATED_ERROR_KINDS = new Set([
+    'dns',
+    'dns-transient',
+    'local-network',
+    'connection-timeout',
+    'connection-reset',
+]);

package/lib/types.d.ts

@@ -10,7 +10,7 @@ import type { CrawlerError, PageData } from './utils/types/types.js';
  * documented per-property rather than scattered across the two emit sites.
  */
 export interface InventoryRunAggregates {
-    /** Total non-empty lines in the input list (= `inventoryUrls.length` before any filtering). Stored verbatim as `inventory_runs.total_lines`. */
+    /** `inventoryUrls.length` as received by `CrawlerOrchestrator.inventory` — the CLI (`inventoryCrawl`) has already warned-and-dropped unparseable-URL lines before this point, so this counts valid URLs, not raw source-file lines. Stored verbatim as `inventory_runs.total_lines`. */
     inventoryUrlsCount: number;
     /** Number of novel URLs classified as HTML and queued for render. Stored as `new_pages` (excludes anchor-discovered descendants — those add later via the crawler graph and are NOT counted here). */
     htmlSeedsCount: number;
@@ -32,6 +32,14 @@ export interface InventoryRunAggregates {
      * `source_file_sha256` will be `NULL`.
      */
     sourceFileSha256: string | null;
+    /**
+     * Number of source-file lines the CLI warned-and-dropped for failing
+     * URL validation, before `inventoryUrlsCount` was ever counted. Stored
+     * verbatim as `inventory_runs.invalid_skipped`. `null` for programmatic
+     * callers that built `inventoryUrls` in-memory — there is no source
+     * file, so no line was ever dropped as invalid.
+     */
+    invalidSkipped: number | null;
 }
 /**
  * Coarse cause of a crawl/scrape failure.

package/lib/utils/array/each-splitted.d.ts

@@ -7,4 +7,4 @@
  * @param callback - A function to invoke on each chunk. May be synchronous or asynchronous.
  * @returns A promise that resolves when all chunk callbacks have completed.
  */
-export declare function eachSplitted<T>(a: T[], count: number, callback: (items: T[]) => void | Promise<void>): Promise<void>;
+export declare function eachSplitted<T>(a: readonly T[], count: number, callback: (items: T[]) => void | Promise<void>): Promise<void>;

package/lib/utils/compute-file-sha256.d.ts

@@ -1,24 +1,24 @@
 /**
- * Compute the SHA-256 digest of a file's bytes, streaming so memory stays
- * O(1) regardless of file size.
+ * Compute the SHA-256 digest of an in-memory byte buffer.
  *
- * Used by `CrawlerOrchestrator.inventory` to fingerprint the source
- * URL list `.txt` and store the digest on the `inventory_runs` row. The
- * digest is the content-identity key for "was this exact list applied
- * before" — recorded for operator audit; nothing dedupes on it at write
- * time.
+ * Used by `CrawlerOrchestrator.inventory` to fingerprint the source URL
+ * list — the digest is both recorded on the `inventory_runs` audit row
+ * (the content-identity key for "was this exact list applied before") and
+ * used as the file name under which the raw list is archived
+ * (`Archive.saveInventorySourceList`).
  *
- * Returns `null` instead of throwing when the file cannot be read (e.g.
- * vanished mid-run, permissions issue) so the inventory run's audit row
- * can still be written with a NULL digest. Hashing failure is an audit
- * loss, not a correctness failure — the actual ingestion has already
- * succeeded by the time this is called.
- * @param filePath - Absolute or relative path to the file to hash.
- * @returns Lower-case hex digest (64 chars), or `null` if reading failed.
+ * Takes an already-read `Buffer` rather than a file path: the CLI reads the
+ * source file exactly once (`inventoryCrawl`) and derives the digest, the
+ * parsed URL list, and the archived copy all from that single buffer. A
+ * separate read-then-hash pass would let the file change between the two
+ * reads and desync the archived bytes from the hash naming them.
+ * @param bytes - The exact bytes to hash.
+ * @returns Lower-case hex digest (64 chars).
  * @example
  * ```ts
- * const sha = await computeFileSha256('/tmp/list.txt');
- * if (sha) console.log(sha); // 64-char hex string
+ * const bytes = await fs.readFile('/tmp/list.txt');
+ * const sha = computeFileSha256(bytes);
+ * console.log(sha); // 64-char hex string
  * ```
  */
-export declare function computeFileSha256(filePath: string): Promise<string | null>;
+export declare function computeFileSha256(bytes: Buffer): string;