@nitpicker/crawler 0.9.0 → 0.12.0

package/lib/crawler/is-puppeteer-fallback-candidate.js

@@ -0,0 +1,63 @@
+import { classifyErrorKind } from '../classify-error-kind.js';
+/**
+ * Error kinds where a full puppeteer navigation has a realistic chance of
+ * succeeding even though the HEAD pre-flight (and its GET fallback) failed.
+ *
+ * These are the failure modes that a misconfigured WAF / middlebox / slow
+ * origin tends to produce against a bare HEAD/GET probe while still letting
+ * a real browser through — the browser uses a different request shape (full
+ * navigation lifecycle, JS-capable Accept headers, real cookies, optionally
+ * client TLS hints), and some hostile middleboxes only inspect the cheap
+ * shape. The fallback is one attempt only; if puppeteer also fails the URL
+ * is recorded as `status = -1` like before.
+ *
+ * Excluded kinds:
+ * - **dns / dns-transient** — DNS resolution happens at the OS level before
+ *   any browser request; puppeteer hits the same `getaddrinfo` outcome.
+ * - **tls** — Chromium will refuse the same certificate the Node TLS stack
+ *   refused (expired, wrong SAN, untrusted CA).
+ * - **client-blocked** — by definition the browser is the one rejecting.
+ * - **connection-refused** — TCP RST from the listener; same answer regardless
+ *   of client.
+ * - **connection-timeout** — `ETIMEDOUT` at the TCP connect stage means the
+ *   packets never reached the host (no SYN-ACK); puppeteer issues the same
+ *   `connect()` call and gets the same answer. Reserved for the middlebox
+ *   case (request reached the server, response timed out), which classifies
+ *   as `timeout` via the `NetTimeoutError "Timeout: <url>"` shape.
+ * - **local-network** — operator-side connectivity loss; nothing on this
+ *   machine will reach the host.
+ * - **protocol** — puppeteer lifecycle race; bouncing back to puppeteer
+ *   reproduces the same race.
+ * - **unknown** — by design. Spinning up a fresh Chromium for every
+ *   unclassifiable error is too expensive; if a real-world WAF / middlebox
+ *   pattern lands in `unknown`, add a matcher to {@link classifyErrorKind}
+ *   so it lands in one of the four included kinds above (where the fallback
+ *   has a meaningful chance of succeeding) instead of widening this set.
+ *
+ * `PreloadShortCircuitError`'s synthesised `getaddrinfo ENOTFOUND` message
+ * classifies into `dns` and is therefore filtered out automatically — no
+ * separate instanceof guard is needed at the call site.
+ */
+const PUPPETEER_FALLBACK_KINDS = new Set([
+    'timeout',
+    'connection-reset',
+    'parse-error',
+]);
+/**
+ * Decide whether a failed HEAD/GET pre-flight error message warrants one
+ * puppeteer fallback attempt. Pure and deterministic — the same message
+ * always gives the same answer, so the decision can be unit-tested without
+ * spinning up a browser.
+ * @param message - The pre-flight error message (typically the last rejected
+ *   `retryCall` attempt).
+ * @returns `true` when puppeteer should be tried once, `false` to give up.
+ * @example
+ * ```ts
+ * isPuppeteerFallbackCandidate('Timeout: https://slow.example.org/'); // true
+ * isPuppeteerFallbackCandidate('getaddrinfo ENOTFOUND host.invalid'); // false
+ * isPuppeteerFallbackCandidate('net::ERR_CERT_DATE_INVALID');         // false
+ * ```
+ */
+export function isPuppeteerFallbackCandidate(message) {
+    return PUPPETEER_FALLBACK_KINDS.has(classifyErrorKind(message));
+}

package/lib/crawler/kill-process-tree.d.ts

@@ -0,0 +1,94 @@
+import type { ChildProcess, SpawnOptions } from 'node:child_process';
+/**
+ * Sends a signal to a single process.
+ *
+ * Abstracted from `process.kill` so tests can verify exactly which PIDs are
+ * signalled without actually touching the OS.
+ */
+export interface ProcessKiller {
+    /**
+     * Sends `signal` to the process with the given PID. Implementations should
+     * swallow ESRCH ("no such process") since the target may have already
+     * exited between enumeration and the kill call.
+     * @param pid - The PID to signal.
+     * @param signal - The signal name (e.g. `'SIGKILL'`) or number.
+     */
+    kill(pid: number, signal: NodeJS.Signals | number): void;
+}
+/**
+ * Spawner signature accepted by `KillProcessTreeDeps`. Pins
+ * `child_process.spawn` to the `(command, args, options)` overload — the
+ * bare import is a union of many overloads (including `(command, options)`)
+ * that does not narrow to this shape without a wrapper. Tests can pass a
+ * `vi.fn()` returning a tiny EventEmitter cast to `ChildProcess`.
+ */
+export type Spawner = (command: string, args: readonly string[], options: SpawnOptions) => ChildProcess;
+/**
+ * Debug logger compatible with the `debug` package's printf-style API.
+ *
+ * Called only when a best-effort kill path fails (ESRCH, ENOENT for `ps` or
+ * `taskkill`, non-zero exit code). Production callers should pass the
+ * crawler-namespaced logger so failures show up under
+ * `DEBUG=Nitpicker:Crawler`.
+ */
+export type KillProcessTreeLogger = (
+/** printf-style format string. */
+formatter: string, 
+/** Arguments interpolated into the format string. */
+...args: readonly unknown[]) => void;
+/**
+ * Dependency overrides for {@link killProcessTree}.
+ *
+ * Default implementations shell out to `ps` (POSIX) or `taskkill` (Windows),
+ * which makes them awkward to unit-test directly. Injecting these lets tests
+ * verify the orchestration without invoking real processes.
+ */
+export interface KillProcessTreeDeps {
+    /**
+     * Lists the PIDs of every descendant of `rootPid` in BFS order
+     * (parents before children). Used only on POSIX.
+     */
+    listDescendants?: (rootPid: number) => Promise<readonly number[]>;
+    /** Signals a single PID. Used only on POSIX. */
+    killer?: ProcessKiller;
+    /**
+     * Performs an OS-level tree-kill. Used only on Windows, where `taskkill
+     * /T /F` handles the entire tree atomically.
+     */
+    runTreeKill?: (rootPid: number) => Promise<void>;
+    /** Override for `process.platform` so cross-platform paths can be tested. */
+    platform?: NodeJS.Platform;
+    /**
+     * `child_process.spawn` substitute used when `runTreeKill` is not
+     * supplied (i.e. the default Windows path). Lets tests assert the exact
+     * command + args without mocking the global module.
+     */
+    spawn?: Spawner;
+    /**
+     * Receives a single line per best-effort failure path (ENOENT,
+     * non-zero exit, etc.). Defaults to a no-op so production callers stay
+     * silent unless they opt in.
+     */
+    log?: KillProcessTreeLogger;
+}
+/**
+ * Kills a process and every one of its descendants.
+ *
+ * WHY: After SIGKILL'ing Chromium's parent process, its renderer / network /
+ * zygote subprocesses linger on Linux/macOS because puppeteer spawns Chromium
+ * with `detached: false`, so we cannot use a process-group signal (a
+ * negative-PID kill would also hit our own Node process). Enumerating
+ * descendants via `ps -A -o pid=,ppid=` and signalling each one — leaves
+ * first — closes the orphan gap. On Windows `taskkill /T /F /PID <pid>`
+ * performs the equivalent OS-level tree kill atomically.
+ *
+ * Best-effort: ESRCH (process already gone) and `ps`/`taskkill` invocation
+ * failures are swallowed, so the function never rejects on a partial-kill
+ * outcome. The caller treats this as a hard cleanup that must always
+ * resolve.
+ * @param rootPid - The PID at the root of the tree.
+ * @param signal - The signal to send. Defaults to `'SIGKILL'`. Ignored on
+ *   Windows (`taskkill /F` is always forceful).
+ * @param deps - Test-time overrides.
+ */
+export declare function killProcessTree(rootPid: number, signal?: NodeJS.Signals | number, deps?: KillProcessTreeDeps): Promise<void>;

package/lib/crawler/kill-process-tree.js

@@ -0,0 +1,178 @@
+import { spawn as nodeSpawn } from 'node:child_process';
+/**
+ * Kills a process and every one of its descendants.
+ *
+ * WHY: After SIGKILL'ing Chromium's parent process, its renderer / network /
+ * zygote subprocesses linger on Linux/macOS because puppeteer spawns Chromium
+ * with `detached: false`, so we cannot use a process-group signal (a
+ * negative-PID kill would also hit our own Node process). Enumerating
+ * descendants via `ps -A -o pid=,ppid=` and signalling each one — leaves
+ * first — closes the orphan gap. On Windows `taskkill /T /F /PID <pid>`
+ * performs the equivalent OS-level tree kill atomically.
+ *
+ * Best-effort: ESRCH (process already gone) and `ps`/`taskkill` invocation
+ * failures are swallowed, so the function never rejects on a partial-kill
+ * outcome. The caller treats this as a hard cleanup that must always
+ * resolve.
+ * @param rootPid - The PID at the root of the tree.
+ * @param signal - The signal to send. Defaults to `'SIGKILL'`. Ignored on
+ *   Windows (`taskkill /F` is always forceful).
+ * @param deps - Test-time overrides.
+ */
+export async function killProcessTree(rootPid, signal = 'SIGKILL', deps = {}) {
+    const platform = deps.platform ?? process.platform;
+    const log = deps.log ?? noopLog;
+    const spawner = deps.spawn ?? defaultSpawner;
+    if (platform === 'win32') {
+        const runTreeKill = deps.runTreeKill ?? ((pid) => runWindowsTaskkill(pid, spawner, log));
+        await runTreeKill(rootPid);
+        return;
+    }
+    const listDescendants = deps.listDescendants ?? ((pid) => listPosixDescendants(pid, spawner, log));
+    const killer = deps.killer ?? makePosixDefaultKiller(log);
+    const descendants = await listDescendants(rootPid);
+    // BFS produces parents before children; reverse so leaves die first and
+    // cannot re-spawn anything via their own watchdog before their parent goes.
+    for (const pid of descendants.toReversed()) {
+        killer.kill(pid, signal);
+    }
+    killer.kill(rootPid, signal);
+}
+/** No-op `KillProcessTreeLogger` used when the caller does not supply one. */
+const noopLog = () => { };
+/**
+ * Default {@link Spawner}: a thin wrapper around `node:child_process.spawn`
+ * that pins the call signature to `(command, args, options)`. The bare
+ * `spawn` import has many overloads — including `(command, options)` — so it
+ * is not assignable to our narrower `Spawner` shape; wrapping it removes the
+ * variance without changing runtime behaviour.
+ * @param command
+ * @param args
+ * @param options
+ */
+const defaultSpawner = (command, args, options) => nodeSpawn(command, args, options);
+/**
+ * Builds the default POSIX killer: `process.kill` with ESRCH/EPERM swallowed
+ * and logged via `log` so production callers can observe how often the tree
+ * walk hits already-dead PIDs.
+ * @param log - Receives one line per swallowed error.
+ * @returns A {@link ProcessKiller}.
+ */
+function makePosixDefaultKiller(log) {
+    return {
+        kill(pid, signal) {
+            try {
+                process.kill(pid, signal);
+            }
+            catch (error) {
+                // Already dead (ESRCH) or denied (EPERM) — best-effort.
+                log('process.kill(%d, %s) failed: %O', pid, String(signal), error);
+            }
+        },
+    };
+}
+/**
+ * Walks the process table on POSIX and returns every descendant of
+ * `rootPid` in BFS order.
+ *
+ * Shells out to `ps -A -o pid=,ppid=` via `spawn` (no shell), parses each
+ * row into `(pid, ppid)`, builds a parent-to-children map, then breadth-first
+ * walks from `rootPid`.
+ *
+ * Failure to invoke `ps` (missing binary, permission denied, non-zero exit)
+ * returns an empty array so the caller can still kill the root.
+ * @param rootPid - The PID whose descendants to list.
+ * @param spawner
+ * @param log
+ * @returns A promise resolving to descendant PIDs.
+ */
+async function listPosixDescendants(rootPid, spawner, log) {
+    const parentToChildren = await readPosixProcessMap(spawner, log);
+    const descendants = [];
+    const queue = [rootPid];
+    while (queue.length > 0) {
+        const pid = queue.shift();
+        const children = parentToChildren.get(pid);
+        if (!children)
+            continue;
+        for (const child of children) {
+            descendants.push(child);
+            queue.push(child);
+        }
+    }
+    return descendants;
+}
+/**
+ * Reads the full POSIX process table by spawning `ps` and parsing its output.
+ *
+ * Format requested: `pid=,ppid=` (no headers). Each line is `<pid> <ppid>`.
+ * @param spawner - Substitute for `child_process.spawn`.
+ * @param log - Receives a single line if `ps` cannot be invoked or exits non-zero.
+ * @returns A promise resolving to a parent-PID-to-children map. Empty on
+ *   any invocation failure.
+ */
+async function readPosixProcessMap(spawner, log) {
+    return new Promise((resolve) => {
+        const proc = spawner('ps', ['-A', '-o', 'pid=,ppid='], {
+            stdio: ['ignore', 'pipe', 'ignore'],
+        });
+        let output = '';
+        proc.stdout?.on('data', (chunk) => {
+            output += chunk.toString('utf8');
+        });
+        proc.on('error', (error) => {
+            log('ps invocation failed: %O', error);
+            resolve(new Map());
+        });
+        proc.on('close', (code) => {
+            if (code !== 0) {
+                log('ps exited with non-zero code %s — skipping descendant tree-kill', String(code));
+                resolve(new Map());
+                return;
+            }
+            const map = new Map();
+            for (const line of output.split('\n')) {
+                const match = /^\s*(\d+)\s+(\d+)/.exec(line);
+                if (!match)
+                    continue;
+                const pid = Number(match[1]);
+                const ppid = Number(match[2]);
+                const list = map.get(ppid);
+                if (list) {
+                    list.push(pid);
+                }
+                else {
+                    map.set(ppid, [pid]);
+                }
+            }
+            resolve(map);
+        });
+    });
+}
+/**
+ * Forces a tree-kill on Windows via `taskkill /T /F /PID <pid>`.
+ *
+ * The `/T` flag walks descendants; `/F` forces termination. Any failure
+ * (ENOENT for taskkill, non-zero exit because the PID is already gone) is
+ * logged via `log` and swallowed so the caller's cleanup always resolves.
+ * @param rootPid - The PID at the root of the tree.
+ * @param spawner - Substitute for `child_process.spawn`.
+ * @param log - Receives a single line per failure.
+ */
+async function runWindowsTaskkill(rootPid, spawner, log) {
+    return new Promise((resolve) => {
+        const proc = spawner('taskkill', ['/T', '/F', '/PID', String(rootPid)], {
+            stdio: 'ignore',
+        });
+        proc.on('error', (error) => {
+            log('taskkill invocation failed: %O', error);
+            resolve();
+        });
+        proc.on('close', (code) => {
+            if (code !== 0) {
+                log('taskkill exited with non-zero code %s for PID %d', String(code), rootPid);
+            }
+            resolve();
+        });
+    });
+}

package/lib/crawler/link-list.d.ts

@@ -45,12 +45,32 @@ export default class LinkList {
      * @param resource.page - The scraped page data, if the scrape succeeded.
      * @param resource.error - The error object, if the scrape failed.
      * @param options - URL parsing options (e.g., `disableQueries`).
+     * @param completion - Behaviour overrides for how the redirect chain is folded into
+     *   the done-set.
+     * @param completion.includeRedirectPaths - When `false`, do NOT mark the URLs in
+     *   `resource.page.redirectPaths` as done. The default `true` preserves the
+     *   long-standing behaviour where a redirect chain (`/a → /b → /c`) folds every
+     *   intermediate URL into the done-set in a single sweep — correct for HTTP-layer
+     *   chains because the browser actually followed each hop, so reaching `/b` later
+     *   is a no-op. Pass `false` from the JS-redirect rescue: there `redirectPaths`
+     *   contains a single URL (the JS target Chromium navigated to after `page.goto()`
+     *   returned null), which the browser has NOT yet rendered. Folding it into the
+     *   done-set would make a subsequent `linkList.add(destinationUrl)` no-op (the
+     *   add() guard at line 51-53 refuses keys already in `#done`), so the dest URL
+     *   never enters `#pending` and the dealer never sees a push to enqueue. The
+     *   dealer's own `seen` Set in `#runDeal` (the gate `onPush` consults) is a
+     *   separate registry from `#done` — they are NOT kept in sync — but here it
+     *   does not matter: the rescue's `add()` is what feeds the eventual `enqueue()`
+     *   call, so blocking `add()` alone is enough to silently lose the JS target
+     *   from the archive.
      * @returns The constructed {@link Link} object, or `null` if the URL was not in the queue.
      */
     done(url: ExURL, scope: ReadonlyMap<string, readonly ExURL[]>, resource: {
         page?: PageData;
         error?: Error;
-    }, options: ParseURLOptions): Link | null;
+    }, options: ParseURLOptions, completion?: {
+        includeRedirectPaths?: boolean;
+    }): Link | null;
     /**
      * Get the current pending and in-progress URL lists.
      * @returns An object containing arrays of pending and in-progress URL strings.

package/lib/crawler/link-list.js

@@ -1,6 +1,7 @@
 import { isError } from '@d-zero/beholder';
 import { isLowerLayer } from '@d-zero/shared/is-lower-layer';
 import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
+import { isHtmlContentType } from './is-html-content-type.js';
 import { protocolAgnosticKey } from './protocol-agnostic-key.js';
 /**
  * Manages the queue of URLs discovered during crawling.
@@ -63,9 +64,27 @@ export default class LinkList {
      * @param resource.page - The scraped page data, if the scrape succeeded.
      * @param resource.error - The error object, if the scrape failed.
      * @param options - URL parsing options (e.g., `disableQueries`).
+     * @param completion - Behaviour overrides for how the redirect chain is folded into
+     *   the done-set.
+     * @param completion.includeRedirectPaths - When `false`, do NOT mark the URLs in
+     *   `resource.page.redirectPaths` as done. The default `true` preserves the
+     *   long-standing behaviour where a redirect chain (`/a → /b → /c`) folds every
+     *   intermediate URL into the done-set in a single sweep — correct for HTTP-layer
+     *   chains because the browser actually followed each hop, so reaching `/b` later
+     *   is a no-op. Pass `false` from the JS-redirect rescue: there `redirectPaths`
+     *   contains a single URL (the JS target Chromium navigated to after `page.goto()`
+     *   returned null), which the browser has NOT yet rendered. Folding it into the
+     *   done-set would make a subsequent `linkList.add(destinationUrl)` no-op (the
+     *   add() guard at line 51-53 refuses keys already in `#done`), so the dest URL
+     *   never enters `#pending` and the dealer never sees a push to enqueue. The
+     *   dealer's own `seen` Set in `#runDeal` (the gate `onPush` consults) is a
+     *   separate registry from `#done` — they are NOT kept in sync — but here it
+     *   does not matter: the rescue's `add()` is what feeds the eventual `enqueue()`
+     *   call, so blocking `add()` alone is enough to silently lose the JS target
+     *   from the archive.
      * @returns The constructed {@link Link} object, or `null` if the URL was not in the queue.
      */
-    done(url, scope, resource, options) {
+    done(url, scope, resource, options, completion) {
         const key = protocolAgnosticKey(url.withoutHashAndAuth);
         if (!(this.#pending.has(key) || this.#progress.has(key))) {
             return null;
@@ -95,8 +114,10 @@ export default class LinkList {
                 responseHeaders: resource.page.responseHeaders,
                 title: resource.page.meta.title,
             };
-            for (const path of resource.page.redirectPaths) {
-                urlList.add(protocolAgnosticKey(path));
+            if (completion?.includeRedirectPaths !== false) {
+                for (const path of resource.page.redirectPaths) {
+                    urlList.add(protocolAgnosticKey(path));
+                }
             }
         }
         if (resource.error?.message.includes('ERR_NAME_NOT_RESOLVED')) {
@@ -241,7 +262,7 @@ function isPage(link) {
     if (isError(link.dest.status)) {
         return false;
     }
-    if (link.dest.contentType === 'text/html') {
+    if (isHtmlContentType(link.dest.contentType)) {
         return true;
     }
     return false;

package/lib/crawler/link-to-page-data.d.ts

@@ -2,12 +2,20 @@ import type { Link, PageData } from '../utils/types/types.js';
 /**
  * Convert a {@link Link} object into a {@link PageData} structure.
  *
- * Creates a minimal PageData from the link's destination metadata. This is used
- * when a full scrape is not performed (e.g., for external pages when
- * `fetchExternal` is disabled, or when a scrape error produces a fallback result).
+ * Creates a minimal PageData from the link's destination metadata. This is
+ * used when a full scrape is not performed (e.g., for external pages when
+ * `fetchExternal` is disabled, or when a scrape error produces a fallback
+ * result).
  *
- * Missing destination fields are filled with sensible defaults (e.g., status -1
- * for unknown, empty arrays for anchors/images, empty string for HTML).
+ * Missing destination fields are filled with sensible defaults (e.g.,
+ * status -1 for unknown, empty arrays for anchors/images, empty string for
+ * HTML).
+ *
+ * beholder 3.0.0 elevated several Meta sub-fields from "optional" to
+ * "required array / object": `jsonLd`, `speculationRules`, `tags`, `others`,
+ * `originTrial`. The dummy meta object below populates every required slot
+ * so downstream consumers (database `#insertJsonLd` / `#insertTags`,
+ * `deriveFlatFromMeta`) never hit `undefined` while iterating.
  * @param link - The link to convert, containing URL and optional destination metadata.
  * @returns A PageData object populated from the link's available data.
  */

package/lib/crawler/link-to-page-data.js

@@ -1,12 +1,20 @@
 /**
  * Convert a {@link Link} object into a {@link PageData} structure.
  *
- * Creates a minimal PageData from the link's destination metadata. This is used
- * when a full scrape is not performed (e.g., for external pages when
- * `fetchExternal` is disabled, or when a scrape error produces a fallback result).
+ * Creates a minimal PageData from the link's destination metadata. This is
+ * used when a full scrape is not performed (e.g., for external pages when
+ * `fetchExternal` is disabled, or when a scrape error produces a fallback
+ * result).
  *
- * Missing destination fields are filled with sensible defaults (e.g., status -1
- * for unknown, empty arrays for anchors/images, empty string for HTML).
+ * Missing destination fields are filled with sensible defaults (e.g.,
+ * status -1 for unknown, empty arrays for anchors/images, empty string for
+ * HTML).
+ *
+ * beholder 3.0.0 elevated several Meta sub-fields from "optional" to
+ * "required array / object": `jsonLd`, `speculationRules`, `tags`, `others`,
+ * `originTrial`. The dummy meta object below populates every required slot
+ * so downstream consumers (database `#insertJsonLd` / `#insertTags`,
+ * `deriveFlatFromMeta`) never hit `undefined` while iterating.
  * @param link - The link to convert, containing URL and optional destination metadata.
  * @returns A PageData object populated from the link's available data.
  */
@@ -23,6 +31,19 @@ export function linkToPageData(link) {
         responseHeaders: link.dest?.responseHeaders || null,
         meta: {
             title: link.dest?.title || '',
+            jsonLd: [],
+            speculationRules: [],
+            tags: { detected: {}, entries: [] },
+            others: {
+                meta: {},
+                property: {},
+                httpEquiv: {},
+                itemprop: {},
+                link: [],
+                script: [],
+                iframe: [],
+            },
+            originTrial: [],
         },
         anchorList: [],
         imageList: [],

package/lib/crawler/log-undrained-phase-errors.d.ts

@@ -0,0 +1,37 @@
+import type { BufferedPhaseError } from './drain-phase-errors.js';
+/**
+ * Debug logger compatible with the `debug` package's printf-style API.
+ *
+ * Receives a single line per URL whose buffered phase errors were
+ * undrained at finally time.
+ */
+export type PhaseErrorDropLogger = (
+/** printf-style format string. */
+formatter: string, 
+/** Arguments interpolated into the format string. */
+...args: readonly unknown[]) => void;
+/**
+ * Logs and clears any phase errors still buffered for `urlHref` at
+ * worker-finally time.
+ *
+ * WHY: a Crawler worker's finally clause runs for every code path —
+ * success, hard error, and the early `return` taken for predicted URLs
+ * that were discarded after probing. The success and catch branches
+ * drain via `drainPhaseErrors`, but the predicted-discard branch
+ * skips drain entirely, so any `retryExhausted` events captured during
+ * its probe would silently leak from the Map.
+ *
+ * Calling this function as the final cleanup step:
+ * - Surfaces the drop via `DEBUG=Nitpicker:Crawler` so production runs
+ *   are observable instead of silent.
+ * - Removes the buffer entry so it does not grow across crawls.
+ *
+ * Safe to call after a successful drain: the buffer entry is already
+ * gone and both the log and the delete become no-ops.
+ * @param buffer - The pending-phase-errors map, keyed by URL href.
+ * @param urlHref - URL whose buffer entry should be flushed.
+ * @param log - Receives one line if any undrained errors are present.
+ * @returns The number of phase-error records dropped (0 when nothing
+ *   was buffered).
+ */
+export declare function logUndrainedPhaseErrors(buffer: Map<string, BufferedPhaseError[]>, urlHref: string, log: PhaseErrorDropLogger): number;

package/lib/crawler/log-undrained-phase-errors.js

@@ -0,0 +1,34 @@
+/**
+ * Logs and clears any phase errors still buffered for `urlHref` at
+ * worker-finally time.
+ *
+ * WHY: a Crawler worker's finally clause runs for every code path —
+ * success, hard error, and the early `return` taken for predicted URLs
+ * that were discarded after probing. The success and catch branches
+ * drain via `drainPhaseErrors`, but the predicted-discard branch
+ * skips drain entirely, so any `retryExhausted` events captured during
+ * its probe would silently leak from the Map.
+ *
+ * Calling this function as the final cleanup step:
+ * - Surfaces the drop via `DEBUG=Nitpicker:Crawler` so production runs
+ *   are observable instead of silent.
+ * - Removes the buffer entry so it does not grow across crawls.
+ *
+ * Safe to call after a successful drain: the buffer entry is already
+ * gone and both the log and the delete become no-ops.
+ * @param buffer - The pending-phase-errors map, keyed by URL href.
+ * @param urlHref - URL whose buffer entry should be flushed.
+ * @param log - Receives one line if any undrained errors are present.
+ * @returns The number of phase-error records dropped (0 when nothing
+ *   was buffered).
+ */
+export function logUndrainedPhaseErrors(buffer, urlHref, log) {
+    const remaining = buffer.get(urlHref);
+    if (!remaining || remaining.length === 0) {
+        buffer.delete(urlHref);
+        return 0;
+    }
+    log('Dropped %d phase error(s) for %s (no archive entry created)', remaining.length, urlHref);
+    buffer.delete(urlHref);
+    return remaining.length;
+}

package/lib/crawler/normalize-content-type.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Canonicalizes a Content-Type media type for storage.
+ *
+ * MIME types are case-insensitive (RFC 2045) and may arrive with surrounding
+ * whitespace (e.g. `text/html ` left after stripping the `; charset=...`
+ * parameter). Responses are recorded verbatim (`header.split(';')[0]`) without
+ * normalization, so `Text/HTML` or `text/html ` can otherwise reach the
+ * database. Storing the canonical (trimmed, lower-cased) form lets the exact
+ * SQL page-ness predicate (`WHERE contentType = 'text/html'`) agree with the
+ * code-level {@link isHtmlContentType} check, which trims and lower-cases.
+ * @param contentType - The raw media type, or `null` when unknown.
+ * @returns The trimmed, lower-cased media type, or `null` when unknown/blank.
+ */
+export declare function normalizeContentType(contentType: string | null): string | null;

package/lib/crawler/normalize-content-type.js

@@ -0,0 +1,20 @@
+/**
+ * Canonicalizes a Content-Type media type for storage.
+ *
+ * MIME types are case-insensitive (RFC 2045) and may arrive with surrounding
+ * whitespace (e.g. `text/html ` left after stripping the `; charset=...`
+ * parameter). Responses are recorded verbatim (`header.split(';')[0]`) without
+ * normalization, so `Text/HTML` or `text/html ` can otherwise reach the
+ * database. Storing the canonical (trimmed, lower-cased) form lets the exact
+ * SQL page-ness predicate (`WHERE contentType = 'text/html'`) agree with the
+ * code-level {@link isHtmlContentType} check, which trims and lower-cases.
+ * @param contentType - The raw media type, or `null` when unknown.
+ * @returns The trimmed, lower-cased media type, or `null` when unknown/blank.
+ */
+export function normalizeContentType(contentType) {
+    if (contentType === null) {
+        return null;
+    }
+    const normalized = contentType.trim().toLowerCase();
+    return normalized === '' ? null : normalized;
+}

package/lib/crawler/partition-urls-by-html.d.ts

@@ -0,0 +1,16 @@
+import type { ExURL } from '@d-zero/shared/parse-url';
+/**
+ * Split URLs into a likely-HTML group and a non-HTML group, preserving input
+ * order within each group.
+ *
+ * Used by the crawler's enqueue path to route discovered URLs: likely-HTML URLs
+ * are `unshift`ed to the front of the dealer queue (so page crawling advances
+ * ahead of asset/document fetches) while the rest are `push`ed to the tail.
+ * Keeping the order stable within each group means a batch (e.g. predicted
+ * pagination URLs) stays in ascending order at the front when unshifted as one
+ * call. Classification is delegated to {@link isLikelyHtmlUrl}.
+ * @param urls - The discovered URLs to partition.
+ * @returns A `[html, other]` tuple: likely-HTML URLs and the remainder, each in
+ *   original input order.
+ */
+export declare function partitionUrlsByHtml(urls: readonly ExURL[]): [html: ExURL[], other: ExURL[]];

package/lib/crawler/partition-urls-by-html.js

@@ -0,0 +1,23 @@
+import { isLikelyHtmlUrl } from './is-likely-html-url.js';
+/**
+ * Split URLs into a likely-HTML group and a non-HTML group, preserving input
+ * order within each group.
+ *
+ * Used by the crawler's enqueue path to route discovered URLs: likely-HTML URLs
+ * are `unshift`ed to the front of the dealer queue (so page crawling advances
+ * ahead of asset/document fetches) while the rest are `push`ed to the tail.
+ * Keeping the order stable within each group means a batch (e.g. predicted
+ * pagination URLs) stays in ascending order at the front when unshifted as one
+ * call. Classification is delegated to {@link isLikelyHtmlUrl}.
+ * @param urls - The discovered URLs to partition.
+ * @returns A `[html, other]` tuple: likely-HTML URLs and the remainder, each in
+ *   original input order.
+ */
+export function partitionUrlsByHtml(urls) {
+    const html = [];
+    const other = [];
+    for (const url of urls) {
+        (isLikelyHtmlUrl(url) ? html : other).push(url);
+    }
+    return [html, other];
+}