@nitpicker/crawler 0.14.0 → 0.16.0

package/lib/crawler/probe-network.js

@@ -0,0 +1,21 @@
+import { lookup } from 'node:dns';
+/**
+ * Default {@link NetworkProbe}: an active `dns.lookup` against `host`.
+ *
+ * Deliberately does not hit a hardcoded external address (e.g. `1.1.1.1`) —
+ * probing a host that has already answered successfully during this crawl
+ * session (see `choose-probe-host.ts`) avoids depending on infrastructure
+ * outside the sites actually being crawled. `dns.lookup` alone is enough:
+ * it exercises the operator's local resolver / network path, which is
+ * exactly the layer an operator-side outage breaks.
+ * @param host - Hostname to resolve.
+ * @returns `true` if the lookup succeeds, `false` on any error (including
+ *   NXDOMAIN, timeout, or resolver unavailability).
+ */
+export function probeNetwork(host) {
+    return new Promise((resolve) => {
+        lookup(host, (error) => {
+            resolve(!error);
+        });
+    });
+}

package/lib/crawler/types.d.ts

@@ -1,6 +1,8 @@
+import type { NetworkProbe } from './probe-network.js';
 import type { PageSource } from '../archive/types.js';
+import type { ErrorKind } from '../types.js';
 import type { PageData, CrawlerError, Resource } from '../utils/types/types.js';
-import type { ChangePhaseEvent, ScrapeResult } from '@d-zero/beholder';
+import type { ChangePhaseEvent, ConsoleLogEntry, ScrapeResult } from '@d-zero/beholder';
 import type { ParseURLOptions } from '@d-zero/shared/parse-url';
 /**
  * Result of resolving a URL that redirects to a destination already rendered
@@ -160,6 +162,57 @@ export interface CrawlerOptions extends Required<Pick<ParseURLOptions, 'disableQ
      * the DB DEFAULT `'crawled'` applies.
      */
     inventoryMode: InventoryMode | null;
+    /**
+     * Sliding-window size in ms for network-outage suspicion. See
+     * `NetworkOutageDetector`.
+     */
+    networkOutageWindowMs: number;
+    /**
+     * Minimum error count within {@link networkOutageWindowMs} to declare a
+     * suspect outage.
+     */
+    networkOutageErrorThreshold: number;
+    /**
+     * Minimum distinct-host count within {@link networkOutageWindowMs} to
+     * declare a suspect outage. Guards against a single flaky host looking
+     * like a network-wide event.
+     */
+    networkOutageHostThreshold: number;
+    /**
+     * Interval in ms between recovery probes while the network gate is
+     * closed.
+     */
+    networkOutageProbeIntervalMs: number;
+    /**
+     * Injectable network-reachability probe, or `null` to use the default
+     * `dns.lookup`-based `probeNetwork`. Overriding this is the seam tests
+     * use to simulate confirmed outages and recoveries deterministically
+     * without touching the real network.
+     */
+    networkProbe: NetworkProbe | null;
+    /**
+     * Same-cluster soft-cap threshold (`--dedupe-cap`), or `null` to disable
+     * the feature entirely (the default). When set, {@link Crawler} stops
+     * enqueueing newly-discovered URLs whose shape (see `computeShapeKey`)
+     * has accumulated this many matching-signature observations (see
+     * `DedupeCapTracker`).
+     */
+    dedupeCap: number | null;
+    /**
+     * Hard cap on the number of distinct URL shapes the same-cluster soft
+     * cap tracks at once (`--dedupe-map-cap`); the least-recently-touched
+     * shape is evicted beyond this. Only relevant when {@link dedupeCap} is
+     * non-null.
+     */
+    dedupeMapCap: number;
+    /**
+     * Shape keys already confirmed capped in a prior session
+     * (persisted as `dedupe_cap_events.shape_key`), seeded into the
+     * tracker's sticky set so `--resume` / `--append` / `--retry-failed` /
+     * `--inventory` do not re-admit a trap this crawl already paid the cost
+     * of discovering once. Ignored when {@link dedupeCap} is `null`.
+     */
+    preloadedStickyShapeKeys: readonly string[];
 }
 /**
  * Inventory-mode runtime configuration. Passed from
@@ -294,6 +347,31 @@ export interface CrawlerEventTypes {
         /** The URL of the referenced resource (without hash). */
         src: string;
     };
+    /**
+     * Emitted once per scrape with the console messages / page errors
+     * beholder captured for that page (issue #228). Only emitted when
+     * `entries` is non-empty — see `Crawler#handleConsoleLogs` for why a
+     * degraded re-scrape that captures nothing must not clear prior good
+     * data.
+     */
+    consoleLogs: {
+        /**
+         * The originally-requested URL, normalised (`withoutHashAndAuth`
+         * form) — the same identity `updatePage` resolves its redirect
+         * chain from, NOT necessarily the page that ends up holding the
+         * content.
+         */
+        pageUrl: string;
+        /**
+         * The redirect chain hops captured during fetch, in order. Empty
+         * when the page was not redirected, or when the scrape produced no
+         * `pageData` (a `'skipped'` / `'error'` result) and no redirect
+         * information is available.
+         */
+        redirectPaths: readonly string[];
+        /** The captured console messages / page errors, in capture order. */
+        entries: ConsoleLogEntry[];
+    };
     /**
      * Emitted when the entire crawl process has completed or been aborted.
      */
@@ -349,4 +427,110 @@ export interface CrawlerEventTypes {
          */
         source: PageSource | undefined;
     };
+    /**
+     * Emitted the instant `Crawler` closes its internal network gate after a
+     * recovery probe CONFIRMS a suspect outage (the sliding-window threshold
+     * alone only makes it a suspect — see `NetworkOutageDetector`). The
+     * orchestrator persists this via `Archive.insertNetworkOutage` and must
+     * remember the returned row id to pass to the matching
+     * `networkOutageRecovered` event, since `Crawler` itself never touches
+     * the archive and has no way to know the row's id.
+     */
+    networkOutageConfirmed: {
+        /** Backdated to the earliest error still inside the detector's window at trigger time. */
+        startedAt: number;
+        /** When the sliding window actually crossed both thresholds. */
+        detectedAt: number;
+        /** Hostname the recovery probe is targeting, or `null` if none was available. */
+        probeHost: string | null;
+        triggerErrorCount: number;
+        triggerHostCount: number;
+    };
+    /**
+     * Emitted the instant `Crawler` reopens its internal network gate after
+     * a recovery probe succeeds. NOT emitted when the gate is opened
+     * because the crawl was aborted while paused — in that case the outage
+     * row is deliberately left open for the next writer session's
+     * boot-time finalizer to resolve (see
+     * `db-ops/outages/close-stale-open-network-outages.ts`), since an abort
+     * says nothing about whether the network actually recovered.
+     */
+    networkOutageRecovered: {
+        /** Epoch ms the recovery probe first succeeded. */
+        endedAt: number;
+    };
+    /**
+     * Emitted the instant the opt-in same-cluster soft cap
+     * ({@link CrawlerOptions.dedupeCap}) confirms a URL shape as a trap (see
+     * `DedupeCapTracker`). The orchestrator persists this via
+     * `Archive.insertDedupeCapEvent` and must remember the returned row id so
+     * `rejected_count` can be finalized once at `crawlEnd` (`Crawler` itself
+     * never touches the archive).
+     */
+    dedupeCap: {
+        shapeKey: string;
+        sampleUrl: string;
+        bodyHash: Buffer;
+        effectiveThreshold: number;
+        observedCount: number;
+    };
+}
+/**
+ * Tunables for `NetworkOutageDetector`.
+ */
+export interface NetworkOutageDetectorOptions {
+    /**
+     * Sliding-window size in ms (`W`). Before each check, entries older than
+     * `at - windowMs` (inclusive boundary — an entry exactly `windowMs` old
+     * still counts) are evicted.
+     */
+    readonly windowMs: number;
+    /** Minimum error count within the window to declare a suspect outage (`N`). */
+    readonly errorThreshold: number;
+    /**
+     * Minimum number of DISTINCT hosts represented in the window to declare
+     * a suspect outage (`M`). Guards against one flaky host (a site that is
+     * genuinely retrying/failing on its own) looking like a network-wide
+     * event — a real local-network blip surfaces across unrelated hosts at
+     * once.
+     */
+    readonly hostThreshold: number;
+}
+/**
+ * One observed error, as fed to `NetworkOutageDetector.record`.
+ */
+export interface NetworkErrorRecord {
+    readonly kind: ErrorKind;
+    readonly host: string;
+    /**
+     * Epoch ms this error was observed. Caller-supplied — the detector never
+     * calls `Date.now()` itself, so window-boundary behaviour can be pinned
+     * with exact values instead of fake timers. Callers MUST supply
+     * non-decreasing values across successive `record()` calls; the window
+     * eviction is a simple cutoff against the latest `at` and does not
+     * re-sort out-of-order input.
+     */
+    readonly at: number;
+}
+/**
+ * Emitted by `NetworkOutageDetector.record` the instant the sliding window
+ * crosses both thresholds.
+ */
+export interface OutageSuspect {
+    /**
+     * Backdated to the earliest error still inside the window at trigger
+     * time — NOT the trigger instant itself. A sliding-window detector only
+     * confirms an outage after `W` seconds and `N` errors have accumulated,
+     * so the outage itself started earlier; backdating lets the persisted
+     * `network_outages` row (and the failures it retroactively covers) reach
+     * back to that earlier point instead of losing everything the detector
+     * missed while still accumulating evidence.
+     */
+    readonly startedAt: number;
+    /** The `at` of the record that tripped the threshold. */
+    readonly detectedAt: number;
+    /** Window size at trigger time (== `errorThreshold` or more). */
+    readonly triggerErrorCount: number;
+    /** Distinct host count at trigger time (== `hostThreshold` or more). */
+    readonly triggerHostCount: number;
 }

package/lib/crawler-orchestrator.d.ts

@@ -1,4 +1,5 @@
 import type { Config } from './archive/types.js';
+import type { NetworkProbe } from './crawler/probe-network.js';
 import type { InventoryMode } from './crawler/types.js';
 import type { CrawlEvent } from './types.js';
 import type { ExURL } from '@d-zero/shared/parse-url';
@@ -48,6 +49,38 @@ interface CrawlConfig extends Config {
      * this `null` so new rows are labelled `'crawled'` by the DB DEFAULT.
      */
     inventoryMode: InventoryMode | null;
+    /**
+     * See {@link CrawlerOptions.networkOutageWindowMs}. Omitted (`undefined`
+     * on the `Partial<CrawlConfig>` callers actually pass) falls through to
+     * `Crawler`'s own default — this field exists so tests can shrink the
+     * window for a fast, deterministic outage-detection cycle.
+     */
+    networkOutageWindowMs: number;
+    /** See {@link CrawlerOptions.networkOutageErrorThreshold}. */
+    networkOutageErrorThreshold: number;
+    /** See {@link CrawlerOptions.networkOutageHostThreshold}. */
+    networkOutageHostThreshold: number;
+    /** See {@link CrawlerOptions.networkOutageProbeIntervalMs}. */
+    networkOutageProbeIntervalMs: number;
+    /**
+     * See {@link CrawlerOptions.networkProbe}. The seam tests use to simulate
+     * confirmed outages and recoveries deterministically without touching
+     * the real network — plumbed through from `CrawlerOrchestrator.crawling`'s
+     * `options` so an E2E test can inject it via the public API.
+     */
+    networkProbe: NetworkProbe | null;
+    /** See {@link CrawlerOptions.dedupeCap}. `null`/omitted disables the feature. */
+    dedupeCap: number | null;
+    /** See {@link CrawlerOptions.dedupeMapCap}. Omitted falls through to `Crawler`'s own default. */
+    dedupeMapCap: number;
+    /**
+     * See {@link CrawlerOptions.preloadedStickyShapeKeys}. Set internally by
+     * the four resuming-session static methods
+     * (`append`/`inventory`/`retryFailed`/`resume`), each independently
+     * calling `archive.listDedupeCapShapeKeys()`; not part of the public
+     * options a caller of those methods passes directly.
+     */
+    preloadedStickyShapeKeys: readonly string[];
 }
 /**
  * Callback invoked after the CrawlerOrchestrator instance is fully initialized
@@ -56,6 +89,20 @@ interface CrawlConfig extends Config {
  * @param config - The resolved archive configuration.
  */
 type CrawlInitializedCallback = (orchestrator: CrawlerOrchestrator, config: Config) => void | Promise<void>;
+/**
+ * The CLI's already-read `--inventory` source list, passed to
+ * {@link CrawlerOrchestrator.inventory} instead of a file path — see that
+ * method's `source` param for why the path itself never crosses this
+ * boundary.
+ */
+interface InventorySource {
+    /** Lower-case hex SHA-256 digest of `bytes` (`computeFileSha256(bytes)`). */
+    sha256: string;
+    /** The exact bytes of the source list file, archived verbatim. */
+    bytes: Buffer;
+    /** Number of source-file lines the CLI warned-and-dropped for failing URL validation, before `inventoryUrls` was ever built. Recorded on the audit row as `inventory_runs.invalid_skipped`. */
+    invalidLineCount: number;
+}
 /**
  * The main entry point for Nitpicker web crawling and archiving.
  *
@@ -184,22 +231,29 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * 3. Reject archives with unfinished `pending` URLs — those would inherit
      *    the inventory `source` label by mistake. Operator must resume /
      *    retry-failed first.
-     * 4. Parse the URL list. Anything outside the archived scope is warned
-     *    and skipped (inventory is per-server by design).
-     * 5. Subtract URLs that already exist in `pages` or `resources` so the
+     * 4. If `source` is given, archive its exact bytes under
+     *    `inventory/<sha256>.txt` (see {@link Archive.saveInventorySourceList}).
+     *    Done before scope classification so even a run that discards every
+     *    URL (out of scope or already known) still leaves a recoverable
+     *    copy of what was fed in.
+     * 5. Parse the URL list — the CLI has already warned-and-dropped
+     *    unparseable-URL lines before calling this method, so every
+     *    remaining entry parses. Anything outside the archived scope is
+     *    warned and skipped (inventory is per-server by design).
+     * 6. Subtract URLs that already exist in `pages` or `resources` so the
      *    second (and N-th) inventory pass is a no-op for known rows — keeps
      *    `'inventory-seed'` rows from being silently demoted.
-     * 6. Make `<archive>.bak`. Anything thrown beyond this point restores
+     * 7. Make `<archive>.bak`. Anything thrown beyond this point restores
      *    from the backup.
-     * 7. HEAD-probe each novel URL. Responses classified as HTML are queued
+     * 8. HEAD-probe each novel URL. Responses classified as HTML are queued
      *    as Crawler seeds (`'inventory-seed'`); everything else is recorded
      *    in `resources` directly as `'inventory-seed'` (no browser launch).
-     * 8. If any HTML seeds exist, start a Crawler with
+     * 9. If any HTML seeds exist, start a Crawler with
      *    `inventoryMode = { seedUrls }` so the rendered page and every newly
      *    discovered downstream link is labelled correctly. `resume` is fed
      *    the existing `scraped` / `resources` sets so links into already-
      *    crawled pages stop at the seen-gate without re-rendering.
-     * 9. Drop the backup on success; restore it on any throw.
+     * 10. Drop the backup on success; restore it on any throw.
      *
      * Mutually exclusive with `--append` / `--retry-failed` / `--resume` /
      * `--diff` / `--list` / `--list-file` / `--single` / `--output` — the
@@ -209,20 +263,22 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * @param inventoryUrls - Pre-read URL list (one URL per element).
      * @param options - Optional config overrides — most callers leave this blank and let the archived config flow through.
      * @param initializedCallback - Hook invoked once the orchestrator is constructed but before `crawling` runs (the CLI uses it to attach progress reporting).
-     * @param sourceFileSha256 - **Pre-computed** SHA-256 hex digest of the
-     *   source URL list. The orchestrator deliberately does NOT receive
-     *   the file path: the path is privacy-sensitive (leaks user-home /
-     *   OS structure when archives are shared) and we want it lifted off
-     *   this boundary so no future log line / breadcrumb / error message
-     *   inside the orchestrator can accidentally re-leak it. The CLI
-     *   computes the digest via `computeFileSha256(resolvedListFile)`
-     *   and passes it through here. Pass `null` for programmatic
-     *   callers that built `inventoryUrls` in-memory; the audit row's
-     *   `source_file_sha256` column will be `NULL`.
+     * @param source - The CLI's already-read source list, as `{ sha256, bytes }`.
+     *   The orchestrator deliberately does NOT receive the file path: the
+     *   path is privacy-sensitive (leaks user-home / OS structure when
+     *   archives are shared) and we want it lifted off this boundary so no
+     *   future log line / breadcrumb / error message inside the orchestrator
+     *   can accidentally re-leak it. `bytes` is archived verbatim under
+     *   `inventory/<sha256>.txt` (see {@link Archive.saveInventorySourceList})
+     *   before scope classification, so a later `--inventory` run against
+     *   the same list is an audit no-op even when it discards zero new
+     *   URLs. Pass `null` for programmatic callers that built
+     *   `inventoryUrls` in-memory; the audit row's `source_file_sha256`
+     *   column will be `NULL` and no source list is archived.
      * @returns The orchestrator instance after a successful inventory pass.
      * @throws {Error} When `inventoryUrls` is empty, the archive is in list mode, or pending URLs from a previous crawl remain unresolved.
      */
-    static inventory(archivePath: string, inventoryUrls: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback, sourceFileSha256?: string | null): Promise<CrawlerOrchestrator>;
+    static inventory(archivePath: string, inventoryUrls: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback, source?: InventorySource | null): Promise<CrawlerOrchestrator>;
     /**
      * Re-fetch previously-failed pages in an existing `.nitpicker` archive.
      *