@nitpicker/crawler 0.13.0 → 0.15.0

package/lib/crawler/network-gate.js

@@ -0,0 +1,78 @@
+/**
+ * Level-triggered gate that dealer worker callbacks await before doing any
+ * network work. Achieves "pause the crawl until the network recovers"
+ * without touching `@d-zero/dealer` internals — a closed gate just makes
+ * the worker's task body a long-running `await`, which is all dealer ever
+ * sees.
+ *
+ * **`epoch`** counts outage generations: it increments on every `close()`
+ * transition (open → closed) and holds that value for the remainder of the
+ * closed period AND through the following open period, until the next
+ * `close()`. This is the hook `destinationCache` / `dnsBurnedHostCache`
+ * eviction uses: an entry written while the gate was closed is tagged with
+ * `gate.epoch` at write time; after `open()`, the caller purges entries
+ * tagged with that same epoch value. Because `epoch` only advances on
+ * `close()` (never on `open()`), entries written during ordinary (open)
+ * operation are never mistakenly tagged with a soon-to-be-purged epoch —
+ * only writes that happen while `isOpen` is `false` get tagged at all.
+ */
+export default class NetworkGate {
+    #epoch = 0;
+    #open = true;
+    #waiters = [];
+    /** `true` when the gate is open (the default). */
+    get isOpen() {
+        return this.#open;
+    }
+    /**
+     * Current outage-generation counter. See the class docstring for how
+     * cache-eviction callers are expected to use this value.
+     */
+    get epoch() {
+        return this.#epoch;
+    }
+    /**
+     * Close the gate, blocking future {@link wait} callers until the next
+     * {@link open}. Idempotent — calling `close()` while already closed does
+     * nothing (in particular, it does NOT bump {@link epoch} a second time,
+     * which would otherwise let a still-open outage's cache entries escape
+     * eviction under a stale epoch number).
+     */
+    close() {
+        if (!this.#open) {
+            return;
+        }
+        this.#open = false;
+        this.#epoch += 1;
+    }
+    /**
+     * Open the gate, resolving every {@link wait} caller currently pending —
+     * including ones that subscribed after `close()` but before this call.
+     * Idempotent — calling `open()` while already open does nothing.
+     */
+    open() {
+        if (this.#open) {
+            return;
+        }
+        this.#open = true;
+        const waiters = this.#waiters;
+        this.#waiters = [];
+        for (const resolve of waiters) {
+            resolve();
+        }
+    }
+    /**
+     * Resolve immediately if the gate is open; otherwise resolve on the next
+     * {@link open}. Multiple concurrent callers all resolve on the same
+     * `open()` call — no awaiter is dropped regardless of when it subscribed
+     * relative to others.
+     */
+    wait() {
+        if (this.#open) {
+            return Promise.resolve();
+        }
+        return new Promise((resolve) => {
+            this.#waiters.push(resolve);
+        });
+    }
+}

package/lib/crawler/network-outage-detector.d.ts

@@ -0,0 +1,51 @@
+import type { NetworkErrorRecord, NetworkOutageDetectorOptions, OutageSuspect } from './types.js';
+/**
+ * Sliding-window detector for "this looks like the crawl operator's own
+ * network, not the target sites, is having a bad time" — the trigger that
+ * hands off to an active probe (see `probe-network.ts`) before the crawler
+ * commits to pausing.
+ *
+ * Deliberately dependency-free: no timers, no DB, no `Crawler` instance.
+ * Time flows in via {@link NetworkErrorRecord.at} only, which is what makes
+ * window-boundary behaviour testable with exact values instead of
+ * `vi.useFakeTimers()`.
+ * @example
+ * ```ts
+ * const detector = new NetworkOutageDetector({
+ *   windowMs: 10_000,
+ *   errorThreshold: 5,
+ *   hostThreshold: 2,
+ * });
+ * const suspect = detector.record({ kind: 'dns', host: 'a.example', at: Date.now() });
+ * if (suspect) {
+ *   // probe before closing the gate
+ * }
+ * ```
+ */
+export default class NetworkOutageDetector {
+    #private;
+    constructor(options: NetworkOutageDetectorOptions);
+    /**
+     * Record one observed error and check whether it tips the sliding window
+     * over both thresholds.
+     *
+     * Non-network `kind`s (see `NETWORK_RELATED_ERROR_KINDS`) are silently
+     * ignored: they never enter the window and can never contribute to a
+     * trigger, regardless of how many arrive.
+     * @param record - The observed error.
+     * @param record.kind
+     * @param record.host
+     * @param record.at
+     * @returns An {@link OutageSuspect} the instant both thresholds are met,
+     *   or `null` otherwise. On a trigger, the window is cleared so the same
+     *   batch of errors cannot fire a second time on the next call.
+     */
+    record({ kind, host, at }: NetworkErrorRecord): OutageSuspect | null;
+    /**
+     * Clear the sliding window. Called at the start of a fresh crawl session
+     * (`Crawler.#runDeal`) so error observations from a prior session on the
+     * same `Crawler` instance never carry over — mirrors `#successfulHosts
+     * .clear()` / `#scrapedDestinations.clear()`'s per-session reset.
+     */
+    reset(): void;
+}

package/lib/crawler/network-outage-detector.js

@@ -0,0 +1,81 @@
+import { NETWORK_RELATED_ERROR_KINDS } from '../network-related-error-kinds.js';
+/**
+ * Sliding-window detector for "this looks like the crawl operator's own
+ * network, not the target sites, is having a bad time" — the trigger that
+ * hands off to an active probe (see `probe-network.ts`) before the crawler
+ * commits to pausing.
+ *
+ * Deliberately dependency-free: no timers, no DB, no `Crawler` instance.
+ * Time flows in via {@link NetworkErrorRecord.at} only, which is what makes
+ * window-boundary behaviour testable with exact values instead of
+ * `vi.useFakeTimers()`.
+ * @example
+ * ```ts
+ * const detector = new NetworkOutageDetector({
+ *   windowMs: 10_000,
+ *   errorThreshold: 5,
+ *   hostThreshold: 2,
+ * });
+ * const suspect = detector.record({ kind: 'dns', host: 'a.example', at: Date.now() });
+ * if (suspect) {
+ *   // probe before closing the gate
+ * }
+ * ```
+ */
+export default class NetworkOutageDetector {
+    #entries = [];
+    #errorThreshold;
+    #hostThreshold;
+    #windowMs;
+    constructor(options) {
+        this.#windowMs = options.windowMs;
+        this.#errorThreshold = options.errorThreshold;
+        this.#hostThreshold = options.hostThreshold;
+    }
+    /**
+     * Record one observed error and check whether it tips the sliding window
+     * over both thresholds.
+     *
+     * Non-network `kind`s (see `NETWORK_RELATED_ERROR_KINDS`) are silently
+     * ignored: they never enter the window and can never contribute to a
+     * trigger, regardless of how many arrive.
+     * @param record - The observed error.
+     * @param record.kind
+     * @param record.host
+     * @param record.at
+     * @returns An {@link OutageSuspect} the instant both thresholds are met,
+     *   or `null` otherwise. On a trigger, the window is cleared so the same
+     *   batch of errors cannot fire a second time on the next call.
+     */
+    record({ kind, host, at }) {
+        if (!NETWORK_RELATED_ERROR_KINDS.has(kind)) {
+            return null;
+        }
+        const cutoff = at - this.#windowMs;
+        this.#entries = this.#entries.filter((entry) => entry.at >= cutoff);
+        this.#entries.push({ host, at });
+        const distinctHosts = new Set(this.#entries.map((entry) => entry.host)).size;
+        if (this.#entries.length < this.#errorThreshold ||
+            distinctHosts < this.#hostThreshold) {
+            return null;
+        }
+        const startedAt = Math.min(...this.#entries.map((entry) => entry.at));
+        const suspect = {
+            startedAt,
+            detectedAt: at,
+            triggerErrorCount: this.#entries.length,
+            triggerHostCount: distinctHosts,
+        };
+        this.#entries = [];
+        return suspect;
+    }
+    /**
+     * Clear the sliding window. Called at the start of a fresh crawl session
+     * (`Crawler.#runDeal`) so error observations from a prior session on the
+     * same `Crawler` instance never carry over — mirrors `#successfulHosts
+     * .clear()` / `#scrapedDestinations.clear()`'s per-session reset.
+     */
+    reset() {
+        this.#entries = [];
+    }
+}

package/lib/crawler/network-outage-summary-counter.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Session-scoped counter for how many operator-network outages were
+ * confirmed, and their total duration. The orchestrator accumulates into
+ * this at every `networkOutageRecovered` event, reads it at crawl shutdown
+ * to emit a `[network] N outage(s), Ms total` summary line, then zeroes it
+ * for the next session (mirrors `dnsBurnedHostShortCircuitCounter`'s
+ * lifecycle, including its known limitation: two `CrawlerOrchestrator`
+ * crawls running concurrently in the same process would accumulate into and
+ * zero the same shared counter, cross-contaminating each other's summary.
+ * The CLI only ever runs one crawl per process, so this has never been
+ * observed in practice; fixing it would mean threading per-session state
+ * through both counters together, out of scope here).
+ *
+ * Exposed as a mutable object (not a plain `let`) so the counter remains a
+ * single shared reference across crawler / orchestrator imports — `let`
+ * bindings cannot be mutated from another module.
+ */
+export declare const networkOutageSummaryCounter: {
+    confirmedCount: number;
+    totalDurationMs: number;
+};

package/lib/crawler/network-outage-summary-counter.js

@@ -0,0 +1,18 @@
+/**
+ * Session-scoped counter for how many operator-network outages were
+ * confirmed, and their total duration. The orchestrator accumulates into
+ * this at every `networkOutageRecovered` event, reads it at crawl shutdown
+ * to emit a `[network] N outage(s), Ms total` summary line, then zeroes it
+ * for the next session (mirrors `dnsBurnedHostShortCircuitCounter`'s
+ * lifecycle, including its known limitation: two `CrawlerOrchestrator`
+ * crawls running concurrently in the same process would accumulate into and
+ * zero the same shared counter, cross-contaminating each other's summary.
+ * The CLI only ever runs one crawl per process, so this has never been
+ * observed in practice; fixing it would mean threading per-session state
+ * through both counters together, out of scope here).
+ *
+ * Exposed as a mutable object (not a plain `let`) so the counter remains a
+ * single shared reference across crawler / orchestrator imports — `let`
+ * bindings cannot be mutated from another module.
+ */
+export const networkOutageSummaryCounter = { confirmedCount: 0, totalDurationMs: 0 };

package/lib/crawler/probe-network.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Function shape for an injectable network-reachability probe: given a
+ * hostname, resolve to `true` if the network path to it appears healthy,
+ * `false` otherwise. Never rejects — a probe failure is a normal outcome
+ * (that's the whole point of probing), not an exceptional one.
+ *
+ * Injected via `CrawlerOptions.networkProbe` so tests can simulate
+ * confirmed outages and recoveries deterministically (a function that fails
+ * N times then starts succeeding) without touching the real network — the
+ * same injection pattern as `ResourceLookup` / `PageSourceLookup`.
+ */
+export type NetworkProbe = (host: string) => Promise<boolean>;
+/**
+ * Default {@link NetworkProbe}: an active `dns.lookup` against `host`.
+ *
+ * Deliberately does not hit a hardcoded external address (e.g. `1.1.1.1`) —
+ * probing a host that has already answered successfully during this crawl
+ * session (see `choose-probe-host.ts`) avoids depending on infrastructure
+ * outside the sites actually being crawled. `dns.lookup` alone is enough:
+ * it exercises the operator's local resolver / network path, which is
+ * exactly the layer an operator-side outage breaks.
+ * @param host - Hostname to resolve.
+ * @returns `true` if the lookup succeeds, `false` on any error (including
+ *   NXDOMAIN, timeout, or resolver unavailability).
+ */
+export declare function probeNetwork(host: string): Promise<boolean>;

package/lib/crawler/probe-network.js

@@ -0,0 +1,21 @@
+import { lookup } from 'node:dns';
+/**
+ * Default {@link NetworkProbe}: an active `dns.lookup` against `host`.
+ *
+ * Deliberately does not hit a hardcoded external address (e.g. `1.1.1.1`) —
+ * probing a host that has already answered successfully during this crawl
+ * session (see `choose-probe-host.ts`) avoids depending on infrastructure
+ * outside the sites actually being crawled. `dns.lookup` alone is enough:
+ * it exercises the operator's local resolver / network path, which is
+ * exactly the layer an operator-side outage breaks.
+ * @param host - Hostname to resolve.
+ * @returns `true` if the lookup succeeds, `false` on any error (including
+ *   NXDOMAIN, timeout, or resolver unavailability).
+ */
+export function probeNetwork(host) {
+    return new Promise((resolve) => {
+        lookup(host, (error) => {
+            resolve(!error);
+        });
+    });
+}

package/lib/crawler/resource-to-page-data.js

@@ -54,6 +54,8 @@ export function resourceToPageData(params) {
         anchorList: [],
         imageList: [],
         html: '',
+        mainContents: null,
+        scrollHeight: null,
         isSkipped: false,
     };
 }

package/lib/crawler/types.d.ts

@@ -1,6 +1,8 @@
+import type { NetworkProbe } from './probe-network.js';
 import type { PageSource } from '../archive/types.js';
+import type { ErrorKind } from '../types.js';
 import type { PageData, CrawlerError, Resource } from '../utils/types/types.js';
-import type { ChangePhaseEvent, ScrapeResult } from '@d-zero/beholder';
+import type { ChangePhaseEvent, ConsoleLogEntry, ScrapeResult } from '@d-zero/beholder';
 import type { ParseURLOptions } from '@d-zero/shared/parse-url';
 /**
  * Result of resolving a URL that redirects to a destination already rendered
@@ -129,6 +131,11 @@ export interface CrawlerOptions extends Required<Pick<ParseURLOptions, 'disableQ
     userAgent: string;
     /** Whether to ignore robots.txt restrictions. */
     ignoreRobots: boolean;
+    /**
+     * CSS selector overriding beholder's automatic main-content-region
+     * detection, or `null`/undefined to use the automatic heuristic.
+     */
+    mainContentSelector?: string | null;
     /**
      * Lookup for previously captured sub-resources, or `null` to disable the
      * resource-reuse optimization. See {@link ResourceLookup}.
@@ -155,6 +162,34 @@ export interface CrawlerOptions extends Required<Pick<ParseURLOptions, 'disableQ
      * the DB DEFAULT `'crawled'` applies.
      */
     inventoryMode: InventoryMode | null;
+    /**
+     * Sliding-window size in ms for network-outage suspicion. See
+     * `NetworkOutageDetector`.
+     */
+    networkOutageWindowMs: number;
+    /**
+     * Minimum error count within {@link networkOutageWindowMs} to declare a
+     * suspect outage.
+     */
+    networkOutageErrorThreshold: number;
+    /**
+     * Minimum distinct-host count within {@link networkOutageWindowMs} to
+     * declare a suspect outage. Guards against a single flaky host looking
+     * like a network-wide event.
+     */
+    networkOutageHostThreshold: number;
+    /**
+     * Interval in ms between recovery probes while the network gate is
+     * closed.
+     */
+    networkOutageProbeIntervalMs: number;
+    /**
+     * Injectable network-reachability probe, or `null` to use the default
+     * `dns.lookup`-based `probeNetwork`. Overriding this is the seam tests
+     * use to simulate confirmed outages and recoveries deterministically
+     * without touching the real network.
+     */
+    networkProbe: NetworkProbe | null;
 }
 /**
  * Inventory-mode runtime configuration. Passed from
@@ -289,6 +324,31 @@ export interface CrawlerEventTypes {
         /** The URL of the referenced resource (without hash). */
         src: string;
     };
+    /**
+     * Emitted once per scrape with the console messages / page errors
+     * beholder captured for that page (issue #228). Only emitted when
+     * `entries` is non-empty — see `Crawler#handleConsoleLogs` for why a
+     * degraded re-scrape that captures nothing must not clear prior good
+     * data.
+     */
+    consoleLogs: {
+        /**
+         * The originally-requested URL, normalised (`withoutHashAndAuth`
+         * form) — the same identity `updatePage` resolves its redirect
+         * chain from, NOT necessarily the page that ends up holding the
+         * content.
+         */
+        pageUrl: string;
+        /**
+         * The redirect chain hops captured during fetch, in order. Empty
+         * when the page was not redirected, or when the scrape produced no
+         * `pageData` (a `'skipped'` / `'error'` result) and no redirect
+         * information is available.
+         */
+        redirectPaths: readonly string[];
+        /** The captured console messages / page errors, in capture order. */
+        entries: ConsoleLogEntry[];
+    };
     /**
      * Emitted when the entire crawl process has completed or been aborted.
      */
@@ -344,4 +404,95 @@ export interface CrawlerEventTypes {
          */
         source: PageSource | undefined;
     };
+    /**
+     * Emitted the instant `Crawler` closes its internal network gate after a
+     * recovery probe CONFIRMS a suspect outage (the sliding-window threshold
+     * alone only makes it a suspect — see `NetworkOutageDetector`). The
+     * orchestrator persists this via `Archive.insertNetworkOutage` and must
+     * remember the returned row id to pass to the matching
+     * `networkOutageRecovered` event, since `Crawler` itself never touches
+     * the archive and has no way to know the row's id.
+     */
+    networkOutageConfirmed: {
+        /** Backdated to the earliest error still inside the detector's window at trigger time. */
+        startedAt: number;
+        /** When the sliding window actually crossed both thresholds. */
+        detectedAt: number;
+        /** Hostname the recovery probe is targeting, or `null` if none was available. */
+        probeHost: string | null;
+        triggerErrorCount: number;
+        triggerHostCount: number;
+    };
+    /**
+     * Emitted the instant `Crawler` reopens its internal network gate after
+     * a recovery probe succeeds. NOT emitted when the gate is opened
+     * because the crawl was aborted while paused — in that case the outage
+     * row is deliberately left open for the next writer session's
+     * boot-time finalizer to resolve (see
+     * `db-ops/outages/close-stale-open-network-outages.ts`), since an abort
+     * says nothing about whether the network actually recovered.
+     */
+    networkOutageRecovered: {
+        /** Epoch ms the recovery probe first succeeded. */
+        endedAt: number;
+    };
+}
+/**
+ * Tunables for `NetworkOutageDetector`.
+ */
+export interface NetworkOutageDetectorOptions {
+    /**
+     * Sliding-window size in ms (`W`). Before each check, entries older than
+     * `at - windowMs` (inclusive boundary — an entry exactly `windowMs` old
+     * still counts) are evicted.
+     */
+    readonly windowMs: number;
+    /** Minimum error count within the window to declare a suspect outage (`N`). */
+    readonly errorThreshold: number;
+    /**
+     * Minimum number of DISTINCT hosts represented in the window to declare
+     * a suspect outage (`M`). Guards against one flaky host (a site that is
+     * genuinely retrying/failing on its own) looking like a network-wide
+     * event — a real local-network blip surfaces across unrelated hosts at
+     * once.
+     */
+    readonly hostThreshold: number;
+}
+/**
+ * One observed error, as fed to `NetworkOutageDetector.record`.
+ */
+export interface NetworkErrorRecord {
+    readonly kind: ErrorKind;
+    readonly host: string;
+    /**
+     * Epoch ms this error was observed. Caller-supplied — the detector never
+     * calls `Date.now()` itself, so window-boundary behaviour can be pinned
+     * with exact values instead of fake timers. Callers MUST supply
+     * non-decreasing values across successive `record()` calls; the window
+     * eviction is a simple cutoff against the latest `at` and does not
+     * re-sort out-of-order input.
+     */
+    readonly at: number;
+}
+/**
+ * Emitted by `NetworkOutageDetector.record` the instant the sliding window
+ * crosses both thresholds.
+ */
+export interface OutageSuspect {
+    /**
+     * Backdated to the earliest error still inside the window at trigger
+     * time — NOT the trigger instant itself. A sliding-window detector only
+     * confirms an outage after `W` seconds and `N` errors have accumulated,
+     * so the outage itself started earlier; backdating lets the persisted
+     * `network_outages` row (and the failures it retroactively covers) reach
+     * back to that earlier point instead of losing everything the detector
+     * missed while still accumulating evidence.
+     */
+    readonly startedAt: number;
+    /** The `at` of the record that tripped the threshold. */
+    readonly detectedAt: number;
+    /** Window size at trigger time (== `errorThreshold` or more). */
+    readonly triggerErrorCount: number;
+    /** Distinct host count at trigger time (== `hostThreshold` or more). */
+    readonly triggerHostCount: number;
 }

package/lib/crawler-orchestrator.d.ts

@@ -1,4 +1,5 @@
 import type { Config } from './archive/types.js';
+import type { NetworkProbe } from './crawler/probe-network.js';
 import type { InventoryMode } from './crawler/types.js';
 import type { CrawlEvent } from './types.js';
 import type { ExURL } from '@d-zero/shared/parse-url';
@@ -48,6 +49,26 @@ interface CrawlConfig extends Config {
      * this `null` so new rows are labelled `'crawled'` by the DB DEFAULT.
      */
     inventoryMode: InventoryMode | null;
+    /**
+     * See {@link CrawlerOptions.networkOutageWindowMs}. Omitted (`undefined`
+     * on the `Partial<CrawlConfig>` callers actually pass) falls through to
+     * `Crawler`'s own default — this field exists so tests can shrink the
+     * window for a fast, deterministic outage-detection cycle.
+     */
+    networkOutageWindowMs: number;
+    /** See {@link CrawlerOptions.networkOutageErrorThreshold}. */
+    networkOutageErrorThreshold: number;
+    /** See {@link CrawlerOptions.networkOutageHostThreshold}. */
+    networkOutageHostThreshold: number;
+    /** See {@link CrawlerOptions.networkOutageProbeIntervalMs}. */
+    networkOutageProbeIntervalMs: number;
+    /**
+     * See {@link CrawlerOptions.networkProbe}. The seam tests use to simulate
+     * confirmed outages and recoveries deterministically without touching
+     * the real network — plumbed through from `CrawlerOrchestrator.crawling`'s
+     * `options` so an E2E test can inject it via the public API.
+     */
+    networkProbe: NetworkProbe | null;
 }
 /**
  * Callback invoked after the CrawlerOrchestrator instance is fully initialized
@@ -56,6 +77,20 @@ interface CrawlConfig extends Config {
  * @param config - The resolved archive configuration.
  */
 type CrawlInitializedCallback = (orchestrator: CrawlerOrchestrator, config: Config) => void | Promise<void>;
+/**
+ * The CLI's already-read `--inventory` source list, passed to
+ * {@link CrawlerOrchestrator.inventory} instead of a file path — see that
+ * method's `source` param for why the path itself never crosses this
+ * boundary.
+ */
+interface InventorySource {
+    /** Lower-case hex SHA-256 digest of `bytes` (`computeFileSha256(bytes)`). */
+    sha256: string;
+    /** The exact bytes of the source list file, archived verbatim. */
+    bytes: Buffer;
+    /** Number of source-file lines the CLI warned-and-dropped for failing URL validation, before `inventoryUrls` was ever built. Recorded on the audit row as `inventory_runs.invalid_skipped`. */
+    invalidLineCount: number;
+}
 /**
  * The main entry point for Nitpicker web crawling and archiving.
  *
@@ -184,22 +219,29 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * 3. Reject archives with unfinished `pending` URLs — those would inherit
      *    the inventory `source` label by mistake. Operator must resume /
      *    retry-failed first.
-     * 4. Parse the URL list. Anything outside the archived scope is warned
-     *    and skipped (inventory is per-server by design).
-     * 5. Subtract URLs that already exist in `pages` or `resources` so the
+     * 4. If `source` is given, archive its exact bytes under
+     *    `inventory/<sha256>.txt` (see {@link Archive.saveInventorySourceList}).
+     *    Done before scope classification so even a run that discards every
+     *    URL (out of scope or already known) still leaves a recoverable
+     *    copy of what was fed in.
+     * 5. Parse the URL list — the CLI has already warned-and-dropped
+     *    unparseable-URL lines before calling this method, so every
+     *    remaining entry parses. Anything outside the archived scope is
+     *    warned and skipped (inventory is per-server by design).
+     * 6. Subtract URLs that already exist in `pages` or `resources` so the
      *    second (and N-th) inventory pass is a no-op for known rows — keeps
      *    `'inventory-seed'` rows from being silently demoted.
-     * 6. Make `<archive>.bak`. Anything thrown beyond this point restores
+     * 7. Make `<archive>.bak`. Anything thrown beyond this point restores
      *    from the backup.
-     * 7. HEAD-probe each novel URL. Responses classified as HTML are queued
+     * 8. HEAD-probe each novel URL. Responses classified as HTML are queued
      *    as Crawler seeds (`'inventory-seed'`); everything else is recorded
      *    in `resources` directly as `'inventory-seed'` (no browser launch).
-     * 8. If any HTML seeds exist, start a Crawler with
+     * 9. If any HTML seeds exist, start a Crawler with
      *    `inventoryMode = { seedUrls }` so the rendered page and every newly
      *    discovered downstream link is labelled correctly. `resume` is fed
      *    the existing `scraped` / `resources` sets so links into already-
      *    crawled pages stop at the seen-gate without re-rendering.
-     * 9. Drop the backup on success; restore it on any throw.
+     * 10. Drop the backup on success; restore it on any throw.
      *
      * Mutually exclusive with `--append` / `--retry-failed` / `--resume` /
      * `--diff` / `--list` / `--list-file` / `--single` / `--output` — the
@@ -209,20 +251,22 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * @param inventoryUrls - Pre-read URL list (one URL per element).
      * @param options - Optional config overrides — most callers leave this blank and let the archived config flow through.
      * @param initializedCallback - Hook invoked once the orchestrator is constructed but before `crawling` runs (the CLI uses it to attach progress reporting).
-     * @param sourceFileSha256 - **Pre-computed** SHA-256 hex digest of the
-     *   source URL list. The orchestrator deliberately does NOT receive
-     *   the file path: the path is privacy-sensitive (leaks user-home /
-     *   OS structure when archives are shared) and we want it lifted off
-     *   this boundary so no future log line / breadcrumb / error message
-     *   inside the orchestrator can accidentally re-leak it. The CLI
-     *   computes the digest via `computeFileSha256(resolvedListFile)`
-     *   and passes it through here. Pass `null` for programmatic
-     *   callers that built `inventoryUrls` in-memory; the audit row's
-     *   `source_file_sha256` column will be `NULL`.
+     * @param source - The CLI's already-read source list, as `{ sha256, bytes }`.
+     *   The orchestrator deliberately does NOT receive the file path: the
+     *   path is privacy-sensitive (leaks user-home / OS structure when
+     *   archives are shared) and we want it lifted off this boundary so no
+     *   future log line / breadcrumb / error message inside the orchestrator
+     *   can accidentally re-leak it. `bytes` is archived verbatim under
+     *   `inventory/<sha256>.txt` (see {@link Archive.saveInventorySourceList})
+     *   before scope classification, so a later `--inventory` run against
+     *   the same list is an audit no-op even when it discards zero new
+     *   URLs. Pass `null` for programmatic callers that built
+     *   `inventoryUrls` in-memory; the audit row's `source_file_sha256`
+     *   column will be `NULL` and no source list is archived.
      * @returns The orchestrator instance after a successful inventory pass.
      * @throws {Error} When `inventoryUrls` is empty, the archive is in list mode, or pending URLs from a previous crawl remain unresolved.
      */
-    static inventory(archivePath: string, inventoryUrls: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback, sourceFileSha256?: string | null): Promise<CrawlerOrchestrator>;
+    static inventory(archivePath: string, inventoryUrls: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback, source?: InventorySource | null): Promise<CrawlerOrchestrator>;
     /**
      * Re-fetch previously-failed pages in an existing `.nitpicker` archive.
      *