@nitpicker/crawler 0.9.0 → 0.12.0

package/lib/archive/types.d.ts

@@ -46,6 +46,61 @@ export interface Config extends Required<Pick<ParseURLOptions, 'disableQueries'>
     /** Whether robots.txt restrictions were ignored during crawling. */
     ignoreRobots: boolean;
 }
+/**
+ * Provenance of a page or resource row — which crawler channel originally
+ * inserted it. Stored as `pages.source` / `resources.source` in the
+ * SQLite schema (NOT NULL DEFAULT `'crawled'`).
+ *
+ * - `'crawled'` — discovered via the recursive crawl rooted at `info.roots`.
+ *   Default for pre-`--inventory` archives after the
+ *   `migratePagesResourcesSource` runtime migration.
+ * - `'inventory-seed'` — supplied directly by a `crawl --inventory` URL
+ *   list. For pages this is the HTML URL that was rendered; for resources
+ *   this is a non-HTML URL handed in by the list (HEAD-fetched without
+ *   rendering).
+ * - `'inventory-discovered'` — found by following links from an
+ *   `inventory-seed` page, OR (for resources) loaded by puppeteer while
+ *   rendering one of those pages.
+ *
+ * Used by the viewer as a badge and to indicate why a row was added.
+ * Isolation queries (`listIsolatedPages` / `listUnusedResources`) judge
+ * orphans by `referrer = 0`, NOT by this value — `source` only labels
+ * the row.
+ */
+export type PageSource = 'crawled' | 'inventory-seed' | 'inventory-discovered';
+/**
+ * One row written to the `inventory_runs` audit table on each successful
+ * `--inventory <list>` invocation.
+ *
+ * Schema-mirror interface: every column on `inventory_runs` is represented
+ * here. Only `ran_at` is required — every other field is nullable so the
+ * post-merge raw-SQL backfill path (a one-off `sqlite3 INSERT` for the
+ * initial inventory pass that predated this table) can omit summary
+ * stats it cannot reconstruct.
+ *
+ * The audit log is append-only at Phase 1: there is no UPDATE path, no
+ * UNIQUE constraint on `source_file_sha256`, and no FK to pages /
+ * resources. Phase 2 (`inventory_memberships`) introduces the M:N link
+ * to URLs; Phase 3 (`--refresh`) uses `source_file_sha256` for dedupe.
+ */
+export interface InventoryRunMeta {
+    /** ISO 8601 timestamp at which the run completed (e.g. `'2026-06-21T11:30:00+09:00'`). */
+    ran_at: string;
+    /** Human-readable identifier (e.g. `'prod-2026-06-21'`). `null` when the caller did not supply one. */
+    list_label?: string | null;
+    /** SHA-256 hex digest of the source file. `null` if hashing failed (e.g. file vanished mid-run). */
+    source_file_sha256?: string | null;
+    /** Number of non-empty lines in the input list (= URL count before scope filtering). */
+    total_lines?: number | null;
+    /** Number of new HTML seeds inserted as `pages` rows by this run. */
+    new_pages?: number | null;
+    /** Number of new non-HTML URLs inserted as `resources` rows by this run. */
+    new_resources?: number | null;
+    /** Number of input URLs dropped because they fell outside the archived scope. */
+    scope_skipped?: number | null;
+    /** Free-form text for backfill annotations or operator notes. */
+    notes?: string | null;
+}
 /**
  * Filter type for querying pages from the database.
  *
@@ -59,7 +114,13 @@ export interface Config extends Required<Pick<ParseURLOptions, 'disableQueries'>
  */
 export type PageFilter = 'page' | 'page-included-no-target' | 'external-page' | 'internal-page' | 'no-page' | 'external-no-page' | 'internal-no-page';
 /**
- * Raw database row representing a crawled page in the `pages` table.
+ * Raw database row representing a crawled page in the `pages` table (v2 schema).
+ *
+ * Maps 1:1 to the columns defined by `archive/init-schema.ts`. Most meta
+ * fields are derived from beholder 3.0.0's nested {@link import('@d-zero/beholder').Meta}
+ * via `archive/meta/derive-flat-from-meta.ts` and are stored as plain
+ * scalars for SQL-level filter / projection. The catch-all `meta_extras`
+ * JSON column preserves nested sub-objects not flattened above.
  */
 export interface DB_Page {
     /** Auto-incremented primary key. */
@@ -84,48 +145,118 @@ export interface DB_Page {
     contentLength: number | null;
     /** JSON-serialized HTTP response headers. */
     responseHeaders: string;
-    /** The `lang` attribute value from the HTML element, or null if not present. */
+    /** The `lang` attribute value from the HTML element, or null. */
     lang: string | null;
-    /** The page title from the `<title>` element, or null if not present. */
+    /** The `dir` attribute value, or null. */
+    dir: string | null;
+    /** The `<meta charset>` value, or null. */
+    charset: string | null;
+    /** Absolutised `<base href>`, or null. */
+    baseHref: string | null;
+    /** The raw `<meta name="viewport">` content, or null. */
+    viewport_raw: string | null;
+    /** The primary `<meta name="theme-color">` (no media), or null. */
+    themeColor: string | null;
+    /** `<meta name="application-name">`, or null. */
+    applicationName: string | null;
+    /** `<meta name="author">`, or null. */
+    author: string | null;
+    /** `<meta name="generator">`, or null. */
+    generator: string | null;
+    /** `<meta name="publisher">`, or null. */
+    publisher: string | null;
+    /** The page title from the `<title>` element, or null. */
     title: string | null;
-    /** The meta description content, or null if not present. */
+    /** The meta description content, or null. */
     description: string | null;
-    /** The meta keywords content, or null if not present. */
+    /** The meta keywords content, or null. */
     keywords: string | null;
-    /** Whether the noindex robots directive is set (SQLite INTEGER 0/1). */
-    noindex: number | null;
-    /** Whether the nofollow robots directive is set (SQLite INTEGER 0/1). */
-    nofollow: number | null;
-    /** Whether the noarchive robots directive is set (SQLite INTEGER 0/1). */
-    noarchive: number | null;
-    /** The canonical URL from `<link rel="canonical">`, or null if not present. */
+    /** The raw `<meta name="robots">` content, or null. */
+    robots_raw: string | null;
+    /** Whether the noindex directive is set (SQLite INTEGER 0/1). */
+    robots_noindex: number | null;
+    /** Whether the nofollow directive is set (SQLite INTEGER 0/1). */
+    robots_nofollow: number | null;
+    /** Whether the noarchive directive is set (SQLite INTEGER 0/1). */
+    robots_noarchive: number | null;
+    /** Whether the noimageindex directive is set (SQLite INTEGER 0/1). */
+    robots_noimageindex: number | null;
+    /** `<meta name="googlebot">` content, or null. */
+    googlebot: string | null;
+    /** Absolutised `<link rel="canonical">` href, or null. */
     canonical: string | null;
-    /** The alternate URL from `<link rel="alternate">`, or null if not present. */
-    alternate: string | null;
-    /** The Open Graph type (`og:type`), or null if not present. */
+    /** Absolutised `<link rel="amphtml">` href, or null. */
+    amphtml: string | null;
+    /** Absolutised `<link rel="manifest">` href, or null. */
+    manifest: string | null;
+    /** Absolutised `<link rel="icon">` href, or null. */
+    icon_href: string | null;
+    /** Absolutised `<link rel="apple-touch-icon">` href, or null. */
+    appleTouchIcon_href: string | null;
+    /** og:type, or null. */
     og_type: string | null;
-    /** The Open Graph title (`og:title`), or null if not present. */
+    /** og:title, or null. */
     og_title: string | null;
-    /** The Open Graph site name (`og:site_name`), or null if not present. */
+    /** Absolutised og:url, or null. */
+    og_url: string | null;
+    /** og:site_name, or null. */
     og_site_name: string | null;
-    /** The Open Graph description (`og:description`), or null if not present. */
+    /** og:description, or null. */
     og_description: string | null;
-    /** The Open Graph URL (`og:url`), or null if not present. */
-    og_url: string | null;
-    /** The Open Graph image URL (`og:image`), or null if not present. */
+    /** Absolutised og:image (first if multiple), or null. */
     og_image: string | null;
-    /** The Twitter Card type (`twitter:card`), or null if not present. */
+    /** og:image:alt, or null. */
+    og_image_alt: string | null;
+    /** og:image:width as a string (per spec), or null. */
+    og_image_width: string | null;
+    /** og:image:height as a string (per spec), or null. */
+    og_image_height: string | null;
+    /** og:locale, or null. */
+    og_locale: string | null;
+    /** og:article:published_time, or null. */
+    og_article_published_time: string | null;
+    /** og:article:modified_time, or null. */
+    og_article_modified_time: string | null;
+    /** twitter:card, or null. */
     twitter_card: string | null;
+    /** twitter:site, or null. */
+    twitter_site: string | null;
+    /** twitter:creator, or null. */
+    twitter_creator: string | null;
+    /** twitter:title, or null. */
+    twitter_title: string | null;
+    /** twitter:description, or null. */
+    twitter_description: string | null;
+    /** Absolutised twitter:image (or twitter:image:src fallback), or null. */
+    twitter_image: string | null;
+    /** Facebook app id (`fb:app_id`), or null. */
+    fb_app_id: string | null;
+    /** Google site verification token, or null. */
+    verification_google: string | null;
+    /** `format-detection` telephone (SQLite INTEGER 0/1), or null. */
+    formatDetection_telephone: number | null;
+    /** First time this page row was inserted (UNIX ms), or null on legacy rows. */
+    firstCrawledAt: number | null;
+    /** Last successful re-scrape time (UNIX ms), or null on legacy rows. */
+    lastCrawledAt: number | null;
+    /** Number of `page_tags` rows belonging to this page. */
+    tag_count: number | null;
+    /** `meta.jsonLd.length + meta.speculationRules.length` at scrape time. */
+    jsonld_count: number | null;
+    /** Sorted unique provider names, comma-separated (empty string when none). */
+    tags_providers_csv: string | null;
+    /** JSON-serialised nested Meta sub-objects not flattened above. */
+    meta_extras: string | null;
     /** JSON-serialized network logs captured during scraping, or null if not collected. */
     networkLogs: string | null;
-    /** Relative file path to the saved HTML snapshot, or null if not saved. */
-    html: string | null;
     /** Whether the page was skipped during crawling (1) or processed normally (0). */
     isSkipped: 0 | 1;
     /** The reason the page was skipped, or null if it was not skipped. */
     skipReason: string | null;
     /** The natural URL sort order index, or null if not yet assigned. */
     order: number | null;
+    /** Provenance of the row — see {@link PageSource}. */
+    source: PageSource;
 }
 /**
  * Raw database row representing a redirect relationship.
@@ -283,13 +414,24 @@ export interface DB_Resource {
     cdn: string | 0;
     /** JSON-serialized HTTP response headers, or null if not available. */
     responseHeaders: string | null;
+    /** Provenance of the row — see {@link PageSource}. */
+    source: PageSource;
 }
 /**
  * Connection options for the archive's libsql-backed database.
  */
 export interface DatabaseOption {
-    /** The working directory for the database (used for resolving relative paths). */
-    workingDir: string;
     /** The absolute file path to the SQLite database file. */
     filename: string;
+    /**
+     * When `true`, open the database for read-only inspection:
+     *
+     * - Skip schema migrations (no `ALTER TABLE` / `UPDATE` on the user's file).
+     * - Refuse to mkdir/create the parent dir or the db file — fail loudly if
+     *   either is missing instead of resurrecting them.
+     *
+     * Used by the viewer / MCP server when attaching to an in-progress crawl's
+     * tmpDir, where any write would race the live crawler.
+     */
+    readOnly?: boolean;
 }

package/lib/classify-error-kind.d.ts

@@ -0,0 +1,19 @@
+import type { ErrorKind } from './types.js';
+/**
+ * Classify a raw crawler/scraper error message into a coarse {@link ErrorKind}.
+ *
+ * Pure and deterministic: the same message always yields the same kind, which
+ * is why the kind is derived on read rather than persisted — it can be applied
+ * uniformly to freshly captured `crawl_errors`, legacy `error.log` lines, and
+ * `page_errors` alike.
+ * @param message - The raw error message (a single line is sufficient; the
+ *   cause token such as `ENOTFOUND` or `Navigation timeout` lives there).
+ * @returns The matched kind, or `unknown` when no matcher applies.
+ * @example
+ * ```ts
+ * classifyErrorKind('getaddrinfo ENOTFOUND www.example.com'); // 'dns'
+ * classifyErrorKind('gave up after 3 retries — Race 180,000ms'); // 'timeout'
+ * classifyErrorKind('Protocol error (Page.reload): Target closed'); // 'protocol'
+ * ```
+ */
+export declare function classifyErrorKind(message: string): ErrorKind;

package/lib/classify-error-kind.js

@@ -0,0 +1,122 @@
+/**
+ * Ordered message matchers. The first pattern that matches wins, so more
+ * specific transport causes (DNS, TLS, connection-*) are tested before the
+ * broader `protocol` / `timeout` buckets — e.g. `ETIMEDOUT` must classify as
+ * `connection-timeout`, not the page-level `timeout`, and a puppeteer
+ * `Protocol error` must not be swallowed by the `timeout` matcher.
+ */
+const MATCHERS = [
+    // `dns-transient` must be evaluated before `dns`: an `EAI_AGAIN` line also
+    // carries the `getaddrinfo` token, so the more specific transient pattern
+    // has to win. Splitting it out from `dns` keeps the DNS-burned host cache
+    // (which marks on `kind === 'dns'`) from punishing a host whose only sin
+    // was a local resolver hiccup.
+    { kind: 'dns-transient', pattern: /EAI_AGAIN|\bEREFUSED\b/i },
+    {
+        kind: 'dns',
+        pattern: /ENOTFOUND|getaddrinfo|ERR_NAME_NOT_RESOLVED|ERR_NAME_RESOLUTION_FAILED/i,
+    },
+    {
+        kind: 'tls',
+        // `Hostname/IP does not match certificate's altnames` is Node's
+        // node:tls hostname mismatch error and is emphatically a TLS issue;
+        // adding it here (alongside the OpenSSL / Chromium tokens) keeps
+        // hosts that serve the wrong-name cert (common with misconfigured
+        // edge / load-balancer setups) out of `unknown`. `altnames` is
+        // anchored to the preceding `certificate` token so a request whose
+        // error message merely mentions a path containing the substring
+        // `altnames` (e.g. `https://api.example.com/altnames/lookup` in a
+        // 5xx body) does NOT get misclassified into `tls` (which is a
+        // `PERMANENT_ERROR_KINDS` member — a false-positive would
+        // permanently exclude that page from `--retry-failed`).
+        pattern: /ERR_CERT|ERR_SSL|\bCERT_|SSL routines|ERR_BAD_SSL|UNABLE_TO_VERIFY|unable to verify|self.signed certificate|certificate has expired\s*$|\bERR_TLS|Hostname\/IP does not match certificate|certificate'?s? altnames/i,
+    },
+    { kind: 'connection-refused', pattern: /ECONNREFUSED|ERR_CONNECTION_REFUSED/i },
+    {
+        kind: 'connection-reset',
+        pattern: /ECONNRESET|socket hang up|ERR_CONNECTION_RESET|ERR_CONNECTION_CLOSED|ERR_EMPTY_RESPONSE/i,
+    },
+    {
+        kind: 'connection-timeout',
+        pattern: /ETIMEDOUT|ERR_CONNECTION_TIMED_OUT|ERR_TIMED_OUT/i,
+    },
+    // `local-network` is evaluated AFTER the connection-* matchers so a
+    // concrete cause (refused / reset / timeout) wins when both apply. Only
+    // "local network is unreachable / changed" symptoms — and the OS-level
+    // errors that surface them — land here. Short tokens (`EPIPE`, `EREFUSED`)
+    // are word-bounded so unrelated identifiers don't false-positive.
+    {
+        kind: 'local-network',
+        pattern: /ERR_INTERNET_DISCONNECTED|ERR_NETWORK_CHANGED|ERR_NETWORK_IO_SUSPENDED|ERR_ADDRESS_UNREACHABLE|ERR_NETWORK_UNREACHABLE|ENETUNREACH|EHOSTUNREACH|EADDRNOTAVAIL|ENOTCONN|\bEPIPE\b/i,
+    },
+    {
+        kind: 'parse-error',
+        pattern: /Parse Error|Expected HTTP\/|Unexpected end of stream/i,
+    },
+    // `client-blocked` covers Chromium's ERR_BLOCKED_* family — the browser
+    // actively decided to reject the request (ad/tracker heuristics, CSP,
+    // CORB / ORB, administrator block list, fingerprinting protection,
+    // cleartext policy, …). Per the upstream `net/base/net_error_list.h`,
+    // `ERR_BLOCKED_BY_CLIENT` is documented as "The client chose to block
+    // the request." — i.e. the server was never the deciding party. Listed
+    // before `protocol` so puppeteer's generic "Protocol error" wrapper
+    // (which sometimes embeds the underlying net error code) is correctly
+    // attributed to the blocked layer rather than the protocol layer.
+    {
+        kind: 'client-blocked',
+        pattern: /ERR_BLOCKED_BY_CLIENT|ERR_BLOCKED_BY_ADMINISTRATOR|ERR_BLOCKED_IN_INCOGNITO_BY_ADMINISTRATOR|ERR_BLOCKED_BY_RESPONSE|ERR_BLOCKED_BY_CSP|ERR_BLOCKED_BY_ORB|ERR_BLOCKED_BY_FINGERPRINTING_PROTECTION|ERR_CLEARTEXT_NOT_PERMITTED|ERR_NETWORK_ACCESS_REVOKED/i,
+    },
+    {
+        kind: 'protocol',
+        // `detached frame` is anchored to puppeteer's exact prefix
+        // `Attempted to use detached Frame` (its current Frame.ts
+        // emitter; the `i` flag below catches the lowercase variant
+        // automatically), not the bare two-token substring. The bare
+        // form would match unrelated diagnostics like a console message
+        // "detached frame ref leaked" echoed through a logger. The older
+        // Page-domain `frame (?:was |got )?detached` form is kept as a
+        // separate alternative because Chromium still surfaces that
+        // phrasing in some legacy code paths. Without one of these, the
+        // "Attempted to use detached Frame ..." messages observed on a
+        // real archive would slip into `unknown`.
+        pattern: /Protocol error|Target closed|Session closed|Execution context was destroyed|frame (?:was |got )?detached|Attempted to use detached frame|Navigating frame was detached|Cannot find context|Node with given id|Page\.\w+ returned/i,
+    },
+    {
+        kind: 'timeout',
+        // `Timeout: https?:` matches the NetTimeoutError "Timeout: <url>"
+        // form. Looking for the URL-shaped tail (rather than anchoring at
+        // line start) is what lets us catch the beholder-wrapped variant
+        // `[Retried N times] Timeout: https://...` that gets stored in
+        // `crawl_errors` / `error.log` after retry exhaustion — the bare
+        // `^Timeout:` form would only fire on the immediate failure and
+        // miss every retry-exhausted record (the ones that actually land
+        // in the archive). Required for slow-server timeouts that
+        // previously fell into `unknown`.
+        pattern: /Race \d|Navigation timeout|timeout of \d+\s*ms exceeded|TimeoutError|Timed out|Timeout: https?:/i,
+    },
+];
+/**
+ * Classify a raw crawler/scraper error message into a coarse {@link ErrorKind}.
+ *
+ * Pure and deterministic: the same message always yields the same kind, which
+ * is why the kind is derived on read rather than persisted — it can be applied
+ * uniformly to freshly captured `crawl_errors`, legacy `error.log` lines, and
+ * `page_errors` alike.
+ * @param message - The raw error message (a single line is sufficient; the
+ *   cause token such as `ENOTFOUND` or `Navigation timeout` lives there).
+ * @returns The matched kind, or `unknown` when no matcher applies.
+ * @example
+ * ```ts
+ * classifyErrorKind('getaddrinfo ENOTFOUND www.example.com'); // 'dns'
+ * classifyErrorKind('gave up after 3 retries — Race 180,000ms'); // 'timeout'
+ * classifyErrorKind('Protocol error (Page.reload): Target closed'); // 'protocol'
+ * ```
+ */
+export function classifyErrorKind(message) {
+    for (const { kind, pattern } of MATCHERS) {
+        if (pattern.test(message)) {
+            return kind;
+        }
+    }
+    return 'unknown';
+}

package/lib/crawler/build-js-redirect-edge.d.ts

@@ -0,0 +1,68 @@
+import type { RedirectEdgeResult } from './types.js';
+import type { PageData } from '@d-zero/beholder';
+import type { ExURL } from '@d-zero/shared/parse-url';
+/**
+ * Parameters for {@link buildJsRedirectEdge}.
+ */
+export interface BuildJsRedirectEdgeParams {
+    /** Source URL puppeteer was asked to navigate to. */
+    url: ExURL;
+    /**
+     * Whether `url` is external to the crawl scope. Propagated onto the
+     * synthesised `PageData` so downstream `linkList.done` classifies the
+     * source the same way the original scrape would have.
+     */
+    isExternal: boolean;
+    /**
+     * Raw error message from `BrowserScrapeResult.error`. The rescue only
+     * fires when this matches the narrow `Page.goto returned null` sentinel
+     * (see {@link isJsRedirectErrorShape}) — anything else falls through
+     * so genuine browser failures surface unchanged.
+     */
+    errorMessage: string | null | undefined;
+    /**
+     * Value of `BrowserScrapeResult.postNavigationUrl` — the URL puppeteer's
+     * `page.url()` reported after the throw. Passed to
+     * {@link deriveJsRedirectTarget} for canonicalisation, credential strip,
+     * and scheme filtering.
+     */
+    postNavigationUrl: string | null | undefined;
+    /**
+     * When present, the rescue builds the redirect-edge `PageData` by
+     * spreading this HEAD-pre-flight result and overriding `redirectPaths`.
+     * Use this on the HEAD-success-then-puppeteer-fail path so the source row
+     * carries the real HTTP-level status / content-type from HEAD. When
+     * omitted, the rescue builds a synthetic placeholder via
+     * {@link linkToPageData} with `status = -1` / `statusText = errorMessage`
+     * — used on the HEAD-fail-then-puppeteer-fallback path where there is
+     * no HEAD response to draw from.
+     */
+    headCheckResult?: PageData;
+}
+/**
+ * Build a JS-redirect `RedirectEdgeResult` when the browser-scrape error
+ * looks like puppeteer's `page.goto() === null` shape AND `page.url()`
+ * resolved to a meaningfully-different URL.
+ *
+ * **Why this helper exists** — both rescue call sites in
+ * `Crawler.#scrapePage` (the HEAD-success-then-puppeteer-fail tail and the
+ * HEAD-fail-then-puppeteer-fallback branch inside `#sendHeadRequest`) run
+ * the same three-step recipe: classify the error shape → derive a JS target
+ * from `page.url()` → synthesise a redirect-edge result. Inlining the recipe
+ * twice means a future change to `deriveJsRedirectTarget`'s contract or the
+ * sentinel string has to be applied to both copies in sync; collapsing into
+ * a single helper keeps the rescue's invariants in one place. The
+ * intentional difference between the two call sites (whether to fold a
+ * HEAD result into the synthesised PageData or to start from a
+ * `linkToPageData` placeholder) is reduced to a single optional parameter
+ * (`headCheckResult`), preserving both shapes without behavioural drift.
+ *
+ * The helper logs at `crawlerLog` whenever it fires so operators tailing
+ * `DEBUG=Nitpicker:Crawler` see exactly which URL got rescued and to which
+ * destination — without this trail the rescue is invisible to anyone
+ * debugging "why did this 200-OK source row land in the archive as a 301?".
+ * @param params - Inputs gathered at the call site.
+ * @returns A `RedirectEdgeResult` when the rescue applies, otherwise
+ *   `null` (the caller should fall through to the existing error path).
+ */
+export declare function buildJsRedirectEdge(params: BuildJsRedirectEdgeParams): RedirectEdgeResult | null;

package/lib/crawler/build-js-redirect-edge.js

@@ -0,0 +1,57 @@
+import { crawlerLog } from '../debug.js';
+import { deriveJsRedirectTarget } from './derive-js-redirect-target.js';
+import { isJsRedirectErrorShape } from './is-js-redirect-error-shape.js';
+import { linkToPageData } from './link-to-page-data.js';
+/**
+ * Build a JS-redirect `RedirectEdgeResult` when the browser-scrape error
+ * looks like puppeteer's `page.goto() === null` shape AND `page.url()`
+ * resolved to a meaningfully-different URL.
+ *
+ * **Why this helper exists** — both rescue call sites in
+ * `Crawler.#scrapePage` (the HEAD-success-then-puppeteer-fail tail and the
+ * HEAD-fail-then-puppeteer-fallback branch inside `#sendHeadRequest`) run
+ * the same three-step recipe: classify the error shape → derive a JS target
+ * from `page.url()` → synthesise a redirect-edge result. Inlining the recipe
+ * twice means a future change to `deriveJsRedirectTarget`'s contract or the
+ * sentinel string has to be applied to both copies in sync; collapsing into
+ * a single helper keeps the rescue's invariants in one place. The
+ * intentional difference between the two call sites (whether to fold a
+ * HEAD result into the synthesised PageData or to start from a
+ * `linkToPageData` placeholder) is reduced to a single optional parameter
+ * (`headCheckResult`), preserving both shapes without behavioural drift.
+ *
+ * The helper logs at `crawlerLog` whenever it fires so operators tailing
+ * `DEBUG=Nitpicker:Crawler` see exactly which URL got rescued and to which
+ * destination — without this trail the rescue is invisible to anyone
+ * debugging "why did this 200-OK source row land in the archive as a 301?".
+ * @param params - Inputs gathered at the call site.
+ * @returns A `RedirectEdgeResult` when the rescue applies, otherwise
+ *   `null` (the caller should fall through to the existing error path).
+ */
+export function buildJsRedirectEdge(params) {
+    if (!isJsRedirectErrorShape(params.errorMessage)) {
+        return null;
+    }
+    const jsRedirectTarget = deriveJsRedirectTarget(params.url.withoutHashAndAuth, params.postNavigationUrl);
+    if (jsRedirectTarget === null) {
+        return null;
+    }
+    const pageData = params.headCheckResult
+        ? { ...params.headCheckResult, redirectPaths: [jsRedirectTarget] }
+        : linkToPageData({
+            url: params.url,
+            isExternal: params.isExternal,
+            isLowerLayer: false,
+            dest: {
+                redirectPaths: [jsRedirectTarget],
+                status: -1,
+                statusText: typeof params.errorMessage === 'string' ? params.errorMessage : '',
+                contentType: null,
+                contentLength: null,
+                responseHeaders: null,
+                title: '',
+            },
+        });
+    crawlerLog('JS-redirect rescue fired for %s → %s (HEAD %s)', params.url.href, jsRedirectTarget, params.headCheckResult ? 'available' : 'absent');
+    return { type: 'redirect-edge', source: 'js-redirect', pageData };
+}

package/lib/crawler/build-redirect-event.d.ts

@@ -0,0 +1,24 @@
+import type { CrawlerEventTypes, InventoryMode } from './types.js';
+import type { PageData } from '../utils/types/types.js';
+/**
+ * Build the payload for the {@link CrawlerEventTypes.redirect} event so the
+ * `{ result, source }` shape is constructed in one place. Pure function —
+ * no I/O, no side effects, just the wiring that connects
+ * {@link derivePageSource} to the emit call.
+ *
+ * Exists so a unit test can pin the wiring directly. Before extraction
+ * the assembly lived inline in `Crawler.#scrape`, which meant the only
+ * way to catch "someone deletes the `source` field" or "someone passes
+ * `inventoryMode` where it should pass `derivePageSource(...)`" was the
+ * inventory E2E. Now a single-purpose spec exercises the construction.
+ *
+ * The originating URL passed in `pageUrlWithoutHashAndAuth` must be the
+ * URL the redirect chain STARTS from (the page being scraped), not the
+ * destination — see {@link Database.recordRedirect} JSDoc for why
+ * lineage propagates from origin, not from the destination.
+ * @param pageData - HEAD-resolved page data carrying the redirect chain.
+ * @param inventoryMode - Inventory-mode config from `CrawlerOptions.inventoryMode`, or `null` outside `--inventory`.
+ * @param pageUrlWithoutHashAndAuth - The originating page URL keyed by `withoutHashAndAuth`.
+ * @returns The exact event payload to feed `emit('redirect', ...)`.
+ */
+export declare function buildRedirectEvent(pageData: PageData, inventoryMode: InventoryMode | null, pageUrlWithoutHashAndAuth: string): CrawlerEventTypes['redirect'];

package/lib/crawler/build-redirect-event.js

@@ -0,0 +1,28 @@
+import { derivePageSource } from './derive-page-source.js';
+/**
+ * Build the payload for the {@link CrawlerEventTypes.redirect} event so the
+ * `{ result, source }` shape is constructed in one place. Pure function —
+ * no I/O, no side effects, just the wiring that connects
+ * {@link derivePageSource} to the emit call.
+ *
+ * Exists so a unit test can pin the wiring directly. Before extraction
+ * the assembly lived inline in `Crawler.#scrape`, which meant the only
+ * way to catch "someone deletes the `source` field" or "someone passes
+ * `inventoryMode` where it should pass `derivePageSource(...)`" was the
+ * inventory E2E. Now a single-purpose spec exercises the construction.
+ *
+ * The originating URL passed in `pageUrlWithoutHashAndAuth` must be the
+ * URL the redirect chain STARTS from (the page being scraped), not the
+ * destination — see {@link Database.recordRedirect} JSDoc for why
+ * lineage propagates from origin, not from the destination.
+ * @param pageData - HEAD-resolved page data carrying the redirect chain.
+ * @param inventoryMode - Inventory-mode config from `CrawlerOptions.inventoryMode`, or `null` outside `--inventory`.
+ * @param pageUrlWithoutHashAndAuth - The originating page URL keyed by `withoutHashAndAuth`.
+ * @returns The exact event payload to feed `emit('redirect', ...)`.
+ */
+export function buildRedirectEvent(pageData, inventoryMode, pageUrlWithoutHashAndAuth) {
+    return {
+        result: pageData,
+        source: derivePageSource(inventoryMode, pageUrlWithoutHashAndAuth),
+    };
+}

package/lib/crawler/clear-dns-burned-host-cache.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Clears the DNS-burned host cache and resets the short-circuit counter.
+ * Called between crawl sessions, alongside {@link clearDestinationCache}, to
+ * avoid leaking state from a previous session into a fresh one.
+ */
+export declare function clearDnsBurnedHostCache(): void;

package/lib/crawler/clear-dns-burned-host-cache.js

@@ -0,0 +1,11 @@
+import { dnsBurnedHostCache } from './dns-burned-host-cache.js';
+import { dnsBurnedHostShortCircuitCounter } from './dns-burned-host-short-circuit-counter.js';
+/**
+ * Clears the DNS-burned host cache and resets the short-circuit counter.
+ * Called between crawl sessions, alongside {@link clearDestinationCache}, to
+ * avoid leaking state from a previous session into a fresh one.
+ */
+export function clearDnsBurnedHostCache() {
+    dnsBurnedHostCache.clear();
+    dnsBurnedHostShortCircuitCounter.count = 0;
+}

package/lib/crawler/close-browser-safely.d.ts

@@ -0,0 +1,64 @@
+/**
+ * Minimal structural subset of a Puppeteer `Browser` required to close it safely.
+ *
+ * Declared structurally (rather than importing Puppeteer's `Browser`) so the
+ * function stays unit-testable with a lightweight stub and free of a Puppeteer
+ * import.
+ */
+export interface ClosableBrowser {
+    /** Gracefully closes the browser and all of its pages over the CDP connection. */
+    close(): Promise<void>;
+    /**
+     * Returns the underlying Chromium child process, or `null` when the browser
+     * was connected to (rather than launched) and therefore owns no process.
+     */
+    process(): {
+        /** PID of the Chromium parent process, or `undefined` before spawn settles. */
+        readonly pid?: number;
+        /** Sends a signal to the process; returns whether it was delivered. */
+        kill(signal?: NodeJS.Signals | number): boolean;
+        /** Whether a signal has already been successfully sent to the process. */
+        readonly killed: boolean;
+    } | null;
+}
+/**
+ * Dependency overrides for {@link closeBrowserSafely}. Used only by tests
+ * to substitute the tree-kill orchestration.
+ */
+export interface CloseBrowserSafelyDeps {
+    /**
+     * Kills a process and every descendant. Defaults to {@link killProcessTree}.
+     */
+    killTree?: (pid: number, signal: NodeJS.Signals | number) => Promise<void>;
+}
+/**
+ * Closes a Puppeteer browser, falling back to a hard tree-kill if the graceful
+ * close hangs.
+ *
+ * WHY: When a page's Chromium session dies mid-scrape (e.g. a viewport change
+ * detaches the frame, surfacing `Attempted to use detached Frame` or
+ * `Session closed`), the CDP connection can be left wedged. A bare
+ * `await browser.close()` then never settles, stalling the `deal()` worker and
+ * hanging the whole crawl. Racing the close against a timeout and SIGKILLing
+ * the Chromium process tree (parent + renderer/network/zygote children) on
+ * expiry guarantees the worker always completes and no orphan subprocesses are
+ * left behind.
+ *
+ * The losing timer is cleared explicitly in `.finally()` so it never keeps the
+ * event loop alive after the race settles (a plain `delay()` in `Promise.race`
+ * would leak the timer until it fires).
+ *
+ * The tree-kill happens via {@link killProcessTree}, which enumerates
+ * descendants through `ps` (POSIX) or delegates to `taskkill /T /F` (Windows).
+ * `childProcess.kill('SIGKILL')` is still invoked on the parent up-front
+ * because Node's `ChildProcess.killed` flag governs how Node treats the
+ * spawn handle (reaping etc.); without it the parent would linger in Node's
+ * process table even after the OS-level kill.
+ * @param browser - The browser to close.
+ * @param timeoutMs - Milliseconds to wait for a graceful close before force-killing.
+ *   Defaults to {@link DEFAULT_CLOSE_TIMEOUT_MS}.
+ * @param deps - Test-time overrides (default-free for production callers).
+ * @returns `true` if the graceful close timed out (and a tree-kill was
+ *   attempted), `false` if `close()` settled in time.
+ */
+export declare function closeBrowserSafely(browser: ClosableBrowser, timeoutMs?: number, deps?: CloseBrowserSafelyDeps): Promise<boolean>;