@nitpicker/crawler 0.9.0 → 0.11.0

package/lib/crawler/kill-process-tree.d.ts

@@ -0,0 +1,94 @@
+import type { ChildProcess, SpawnOptions } from 'node:child_process';
+/**
+ * Sends a signal to a single process.
+ *
+ * Abstracted from `process.kill` so tests can verify exactly which PIDs are
+ * signalled without actually touching the OS.
+ */
+export interface ProcessKiller {
+    /**
+     * Sends `signal` to the process with the given PID. Implementations should
+     * swallow ESRCH ("no such process") since the target may have already
+     * exited between enumeration and the kill call.
+     * @param pid - The PID to signal.
+     * @param signal - The signal name (e.g. `'SIGKILL'`) or number.
+     */
+    kill(pid: number, signal: NodeJS.Signals | number): void;
+}
+/**
+ * Spawner signature accepted by `KillProcessTreeDeps`. Pins
+ * `child_process.spawn` to the `(command, args, options)` overload — the
+ * bare import is a union of many overloads (including `(command, options)`)
+ * that does not narrow to this shape without a wrapper. Tests can pass a
+ * `vi.fn()` returning a tiny EventEmitter cast to `ChildProcess`.
+ */
+export type Spawner = (command: string, args: readonly string[], options: SpawnOptions) => ChildProcess;
+/**
+ * Debug logger compatible with the `debug` package's printf-style API.
+ *
+ * Called only when a best-effort kill path fails (ESRCH, ENOENT for `ps` or
+ * `taskkill`, non-zero exit code). Production callers should pass the
+ * crawler-namespaced logger so failures show up under
+ * `DEBUG=Nitpicker:Crawler`.
+ */
+export type KillProcessTreeLogger = (
+/** printf-style format string. */
+formatter: string, 
+/** Arguments interpolated into the format string. */
+...args: readonly unknown[]) => void;
+/**
+ * Dependency overrides for {@link killProcessTree}.
+ *
+ * Default implementations shell out to `ps` (POSIX) or `taskkill` (Windows),
+ * which makes them awkward to unit-test directly. Injecting these lets tests
+ * verify the orchestration without invoking real processes.
+ */
+export interface KillProcessTreeDeps {
+    /**
+     * Lists the PIDs of every descendant of `rootPid` in BFS order
+     * (parents before children). Used only on POSIX.
+     */
+    listDescendants?: (rootPid: number) => Promise<readonly number[]>;
+    /** Signals a single PID. Used only on POSIX. */
+    killer?: ProcessKiller;
+    /**
+     * Performs an OS-level tree-kill. Used only on Windows, where `taskkill
+     * /T /F` handles the entire tree atomically.
+     */
+    runTreeKill?: (rootPid: number) => Promise<void>;
+    /** Override for `process.platform` so cross-platform paths can be tested. */
+    platform?: NodeJS.Platform;
+    /**
+     * `child_process.spawn` substitute used when `runTreeKill` is not
+     * supplied (i.e. the default Windows path). Lets tests assert the exact
+     * command + args without mocking the global module.
+     */
+    spawn?: Spawner;
+    /**
+     * Receives a single line per best-effort failure path (ENOENT,
+     * non-zero exit, etc.). Defaults to a no-op so production callers stay
+     * silent unless they opt in.
+     */
+    log?: KillProcessTreeLogger;
+}
+/**
+ * Kills a process and every one of its descendants.
+ *
+ * WHY: After SIGKILL'ing Chromium's parent process, its renderer / network /
+ * zygote subprocesses linger on Linux/macOS because puppeteer spawns Chromium
+ * with `detached: false`, so we cannot use a process-group signal (a
+ * negative-PID kill would also hit our own Node process). Enumerating
+ * descendants via `ps -A -o pid=,ppid=` and signalling each one — leaves
+ * first — closes the orphan gap. On Windows `taskkill /T /F /PID <pid>`
+ * performs the equivalent OS-level tree kill atomically.
+ *
+ * Best-effort: ESRCH (process already gone) and `ps`/`taskkill` invocation
+ * failures are swallowed, so the function never rejects on a partial-kill
+ * outcome. The caller treats this as a hard cleanup that must always
+ * resolve.
+ * @param rootPid - The PID at the root of the tree.
+ * @param signal - The signal to send. Defaults to `'SIGKILL'`. Ignored on
+ *   Windows (`taskkill /F` is always forceful).
+ * @param deps - Test-time overrides.
+ */
+export declare function killProcessTree(rootPid: number, signal?: NodeJS.Signals | number, deps?: KillProcessTreeDeps): Promise<void>;

package/lib/crawler/kill-process-tree.js

@@ -0,0 +1,178 @@
+import { spawn as nodeSpawn } from 'node:child_process';
+/**
+ * Kills a process and every one of its descendants.
+ *
+ * WHY: After SIGKILL'ing Chromium's parent process, its renderer / network /
+ * zygote subprocesses linger on Linux/macOS because puppeteer spawns Chromium
+ * with `detached: false`, so we cannot use a process-group signal (a
+ * negative-PID kill would also hit our own Node process). Enumerating
+ * descendants via `ps -A -o pid=,ppid=` and signalling each one — leaves
+ * first — closes the orphan gap. On Windows `taskkill /T /F /PID <pid>`
+ * performs the equivalent OS-level tree kill atomically.
+ *
+ * Best-effort: ESRCH (process already gone) and `ps`/`taskkill` invocation
+ * failures are swallowed, so the function never rejects on a partial-kill
+ * outcome. The caller treats this as a hard cleanup that must always
+ * resolve.
+ * @param rootPid - The PID at the root of the tree.
+ * @param signal - The signal to send. Defaults to `'SIGKILL'`. Ignored on
+ *   Windows (`taskkill /F` is always forceful).
+ * @param deps - Test-time overrides.
+ */
+export async function killProcessTree(rootPid, signal = 'SIGKILL', deps = {}) {
+    const platform = deps.platform ?? process.platform;
+    const log = deps.log ?? noopLog;
+    const spawner = deps.spawn ?? defaultSpawner;
+    if (platform === 'win32') {
+        const runTreeKill = deps.runTreeKill ?? ((pid) => runWindowsTaskkill(pid, spawner, log));
+        await runTreeKill(rootPid);
+        return;
+    }
+    const listDescendants = deps.listDescendants ?? ((pid) => listPosixDescendants(pid, spawner, log));
+    const killer = deps.killer ?? makePosixDefaultKiller(log);
+    const descendants = await listDescendants(rootPid);
+    // BFS produces parents before children; reverse so leaves die first and
+    // cannot re-spawn anything via their own watchdog before their parent goes.
+    for (const pid of descendants.toReversed()) {
+        killer.kill(pid, signal);
+    }
+    killer.kill(rootPid, signal);
+}
+/** No-op `KillProcessTreeLogger` used when the caller does not supply one. */
+const noopLog = () => { };
+/**
+ * Default {@link Spawner}: a thin wrapper around `node:child_process.spawn`
+ * that pins the call signature to `(command, args, options)`. The bare
+ * `spawn` import has many overloads — including `(command, options)` — so it
+ * is not assignable to our narrower `Spawner` shape; wrapping it removes the
+ * variance without changing runtime behaviour.
+ * @param command
+ * @param args
+ * @param options
+ */
+const defaultSpawner = (command, args, options) => nodeSpawn(command, args, options);
+/**
+ * Builds the default POSIX killer: `process.kill` with ESRCH/EPERM swallowed
+ * and logged via `log` so production callers can observe how often the tree
+ * walk hits already-dead PIDs.
+ * @param log - Receives one line per swallowed error.
+ * @returns A {@link ProcessKiller}.
+ */
+function makePosixDefaultKiller(log) {
+    return {
+        kill(pid, signal) {
+            try {
+                process.kill(pid, signal);
+            }
+            catch (error) {
+                // Already dead (ESRCH) or denied (EPERM) — best-effort.
+                log('process.kill(%d, %s) failed: %O', pid, String(signal), error);
+            }
+        },
+    };
+}
+/**
+ * Walks the process table on POSIX and returns every descendant of
+ * `rootPid` in BFS order.
+ *
+ * Shells out to `ps -A -o pid=,ppid=` via `spawn` (no shell), parses each
+ * row into `(pid, ppid)`, builds a parent-to-children map, then breadth-first
+ * walks from `rootPid`.
+ *
+ * Failure to invoke `ps` (missing binary, permission denied, non-zero exit)
+ * returns an empty array so the caller can still kill the root.
+ * @param rootPid - The PID whose descendants to list.
+ * @param spawner
+ * @param log
+ * @returns A promise resolving to descendant PIDs.
+ */
+async function listPosixDescendants(rootPid, spawner, log) {
+    const parentToChildren = await readPosixProcessMap(spawner, log);
+    const descendants = [];
+    const queue = [rootPid];
+    while (queue.length > 0) {
+        const pid = queue.shift();
+        const children = parentToChildren.get(pid);
+        if (!children)
+            continue;
+        for (const child of children) {
+            descendants.push(child);
+            queue.push(child);
+        }
+    }
+    return descendants;
+}
+/**
+ * Reads the full POSIX process table by spawning `ps` and parsing its output.
+ *
+ * Format requested: `pid=,ppid=` (no headers). Each line is `<pid> <ppid>`.
+ * @param spawner - Substitute for `child_process.spawn`.
+ * @param log - Receives a single line if `ps` cannot be invoked or exits non-zero.
+ * @returns A promise resolving to a parent-PID-to-children map. Empty on
+ *   any invocation failure.
+ */
+async function readPosixProcessMap(spawner, log) {
+    return new Promise((resolve) => {
+        const proc = spawner('ps', ['-A', '-o', 'pid=,ppid='], {
+            stdio: ['ignore', 'pipe', 'ignore'],
+        });
+        let output = '';
+        proc.stdout?.on('data', (chunk) => {
+            output += chunk.toString('utf8');
+        });
+        proc.on('error', (error) => {
+            log('ps invocation failed: %O', error);
+            resolve(new Map());
+        });
+        proc.on('close', (code) => {
+            if (code !== 0) {
+                log('ps exited with non-zero code %s — skipping descendant tree-kill', String(code));
+                resolve(new Map());
+                return;
+            }
+            const map = new Map();
+            for (const line of output.split('\n')) {
+                const match = /^\s*(\d+)\s+(\d+)/.exec(line);
+                if (!match)
+                    continue;
+                const pid = Number(match[1]);
+                const ppid = Number(match[2]);
+                const list = map.get(ppid);
+                if (list) {
+                    list.push(pid);
+                }
+                else {
+                    map.set(ppid, [pid]);
+                }
+            }
+            resolve(map);
+        });
+    });
+}
+/**
+ * Forces a tree-kill on Windows via `taskkill /T /F /PID <pid>`.
+ *
+ * The `/T` flag walks descendants; `/F` forces termination. Any failure
+ * (ENOENT for taskkill, non-zero exit because the PID is already gone) is
+ * logged via `log` and swallowed so the caller's cleanup always resolves.
+ * @param rootPid - The PID at the root of the tree.
+ * @param spawner - Substitute for `child_process.spawn`.
+ * @param log - Receives a single line per failure.
+ */
+async function runWindowsTaskkill(rootPid, spawner, log) {
+    return new Promise((resolve) => {
+        const proc = spawner('taskkill', ['/T', '/F', '/PID', String(rootPid)], {
+            stdio: 'ignore',
+        });
+        proc.on('error', (error) => {
+            log('taskkill invocation failed: %O', error);
+            resolve();
+        });
+        proc.on('close', (code) => {
+            if (code !== 0) {
+                log('taskkill exited with non-zero code %s for PID %d', String(code), rootPid);
+            }
+            resolve();
+        });
+    });
+}

package/lib/crawler/link-list.js

@@ -1,6 +1,7 @@
 import { isError } from '@d-zero/beholder';
 import { isLowerLayer } from '@d-zero/shared/is-lower-layer';
 import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
+import { isHtmlContentType } from './is-html-content-type.js';
 import { protocolAgnosticKey } from './protocol-agnostic-key.js';
 /**
  * Manages the queue of URLs discovered during crawling.
@@ -241,7 +242,7 @@ function isPage(link) {
     if (isError(link.dest.status)) {
         return false;
     }
-    if (link.dest.contentType === 'text/html') {
+    if (isHtmlContentType(link.dest.contentType)) {
         return true;
     }
     return false;

package/lib/crawler/link-to-page-data.d.ts

@@ -2,12 +2,20 @@ import type { Link, PageData } from '../utils/types/types.js';
 /**
  * Convert a {@link Link} object into a {@link PageData} structure.
  *
- * Creates a minimal PageData from the link's destination metadata. This is used
- * when a full scrape is not performed (e.g., for external pages when
- * `fetchExternal` is disabled, or when a scrape error produces a fallback result).
+ * Creates a minimal PageData from the link's destination metadata. This is
+ * used when a full scrape is not performed (e.g., for external pages when
+ * `fetchExternal` is disabled, or when a scrape error produces a fallback
+ * result).
  *
- * Missing destination fields are filled with sensible defaults (e.g., status -1
- * for unknown, empty arrays for anchors/images, empty string for HTML).
+ * Missing destination fields are filled with sensible defaults (e.g.,
+ * status -1 for unknown, empty arrays for anchors/images, empty string for
+ * HTML).
+ *
+ * beholder 3.0.0 elevated several Meta sub-fields from "optional" to
+ * "required array / object": `jsonLd`, `speculationRules`, `tags`, `others`,
+ * `originTrial`. The dummy meta object below populates every required slot
+ * so downstream consumers (database `#insertJsonLd` / `#insertTags`,
+ * `deriveFlatFromMeta`) never hit `undefined` while iterating.
  * @param link - The link to convert, containing URL and optional destination metadata.
  * @returns A PageData object populated from the link's available data.
  */

package/lib/crawler/link-to-page-data.js

@@ -1,12 +1,20 @@
 /**
  * Convert a {@link Link} object into a {@link PageData} structure.
  *
- * Creates a minimal PageData from the link's destination metadata. This is used
- * when a full scrape is not performed (e.g., for external pages when
- * `fetchExternal` is disabled, or when a scrape error produces a fallback result).
+ * Creates a minimal PageData from the link's destination metadata. This is
+ * used when a full scrape is not performed (e.g., for external pages when
+ * `fetchExternal` is disabled, or when a scrape error produces a fallback
+ * result).
  *
- * Missing destination fields are filled with sensible defaults (e.g., status -1
- * for unknown, empty arrays for anchors/images, empty string for HTML).
+ * Missing destination fields are filled with sensible defaults (e.g.,
+ * status -1 for unknown, empty arrays for anchors/images, empty string for
+ * HTML).
+ *
+ * beholder 3.0.0 elevated several Meta sub-fields from "optional" to
+ * "required array / object": `jsonLd`, `speculationRules`, `tags`, `others`,
+ * `originTrial`. The dummy meta object below populates every required slot
+ * so downstream consumers (database `#insertJsonLd` / `#insertTags`,
+ * `deriveFlatFromMeta`) never hit `undefined` while iterating.
  * @param link - The link to convert, containing URL and optional destination metadata.
  * @returns A PageData object populated from the link's available data.
  */
@@ -23,6 +31,19 @@ export function linkToPageData(link) {
         responseHeaders: link.dest?.responseHeaders || null,
         meta: {
             title: link.dest?.title || '',
+            jsonLd: [],
+            speculationRules: [],
+            tags: { detected: {}, entries: [] },
+            others: {
+                meta: {},
+                property: {},
+                httpEquiv: {},
+                itemprop: {},
+                link: [],
+                script: [],
+                iframe: [],
+            },
+            originTrial: [],
         },
         anchorList: [],
         imageList: [],

package/lib/crawler/log-undrained-phase-errors.d.ts

@@ -0,0 +1,37 @@
+import type { BufferedPhaseError } from './drain-phase-errors.js';
+/**
+ * Debug logger compatible with the `debug` package's printf-style API.
+ *
+ * Receives a single line per URL whose buffered phase errors were
+ * undrained at finally time.
+ */
+export type PhaseErrorDropLogger = (
+/** printf-style format string. */
+formatter: string, 
+/** Arguments interpolated into the format string. */
+...args: readonly unknown[]) => void;
+/**
+ * Logs and clears any phase errors still buffered for `urlHref` at
+ * worker-finally time.
+ *
+ * WHY: a Crawler worker's finally clause runs for every code path —
+ * success, hard error, and the early `return` taken for predicted URLs
+ * that were discarded after probing. The success and catch branches
+ * drain via `drainPhaseErrors`, but the predicted-discard branch
+ * skips drain entirely, so any `retryExhausted` events captured during
+ * its probe would silently leak from the Map.
+ *
+ * Calling this function as the final cleanup step:
+ * - Surfaces the drop via `DEBUG=Nitpicker:Crawler` so production runs
+ *   are observable instead of silent.
+ * - Removes the buffer entry so it does not grow across crawls.
+ *
+ * Safe to call after a successful drain: the buffer entry is already
+ * gone and both the log and the delete become no-ops.
+ * @param buffer - The pending-phase-errors map, keyed by URL href.
+ * @param urlHref - URL whose buffer entry should be flushed.
+ * @param log - Receives one line if any undrained errors are present.
+ * @returns The number of phase-error records dropped (0 when nothing
+ *   was buffered).
+ */
+export declare function logUndrainedPhaseErrors(buffer: Map<string, BufferedPhaseError[]>, urlHref: string, log: PhaseErrorDropLogger): number;

package/lib/crawler/log-undrained-phase-errors.js

@@ -0,0 +1,34 @@
+/**
+ * Logs and clears any phase errors still buffered for `urlHref` at
+ * worker-finally time.
+ *
+ * WHY: a Crawler worker's finally clause runs for every code path —
+ * success, hard error, and the early `return` taken for predicted URLs
+ * that were discarded after probing. The success and catch branches
+ * drain via `drainPhaseErrors`, but the predicted-discard branch
+ * skips drain entirely, so any `retryExhausted` events captured during
+ * its probe would silently leak from the Map.
+ *
+ * Calling this function as the final cleanup step:
+ * - Surfaces the drop via `DEBUG=Nitpicker:Crawler` so production runs
+ *   are observable instead of silent.
+ * - Removes the buffer entry so it does not grow across crawls.
+ *
+ * Safe to call after a successful drain: the buffer entry is already
+ * gone and both the log and the delete become no-ops.
+ * @param buffer - The pending-phase-errors map, keyed by URL href.
+ * @param urlHref - URL whose buffer entry should be flushed.
+ * @param log - Receives one line if any undrained errors are present.
+ * @returns The number of phase-error records dropped (0 when nothing
+ *   was buffered).
+ */
+export function logUndrainedPhaseErrors(buffer, urlHref, log) {
+    const remaining = buffer.get(urlHref);
+    if (!remaining || remaining.length === 0) {
+        buffer.delete(urlHref);
+        return 0;
+    }
+    log('Dropped %d phase error(s) for %s (no archive entry created)', remaining.length, urlHref);
+    buffer.delete(urlHref);
+    return remaining.length;
+}

package/lib/crawler/normalize-content-type.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Canonicalizes a Content-Type media type for storage.
+ *
+ * MIME types are case-insensitive (RFC 2045) and may arrive with surrounding
+ * whitespace (e.g. `text/html ` left after stripping the `; charset=...`
+ * parameter). Responses are recorded verbatim (`header.split(';')[0]`) without
+ * normalization, so `Text/HTML` or `text/html ` can otherwise reach the
+ * database. Storing the canonical (trimmed, lower-cased) form lets the exact
+ * SQL page-ness predicate (`WHERE contentType = 'text/html'`) agree with the
+ * code-level {@link isHtmlContentType} check, which trims and lower-cases.
+ * @param contentType - The raw media type, or `null` when unknown.
+ * @returns The trimmed, lower-cased media type, or `null` when unknown/blank.
+ */
+export declare function normalizeContentType(contentType: string | null): string | null;

package/lib/crawler/normalize-content-type.js

@@ -0,0 +1,20 @@
+/**
+ * Canonicalizes a Content-Type media type for storage.
+ *
+ * MIME types are case-insensitive (RFC 2045) and may arrive with surrounding
+ * whitespace (e.g. `text/html ` left after stripping the `; charset=...`
+ * parameter). Responses are recorded verbatim (`header.split(';')[0]`) without
+ * normalization, so `Text/HTML` or `text/html ` can otherwise reach the
+ * database. Storing the canonical (trimmed, lower-cased) form lets the exact
+ * SQL page-ness predicate (`WHERE contentType = 'text/html'`) agree with the
+ * code-level {@link isHtmlContentType} check, which trims and lower-cases.
+ * @param contentType - The raw media type, or `null` when unknown.
+ * @returns The trimmed, lower-cased media type, or `null` when unknown/blank.
+ */
+export function normalizeContentType(contentType) {
+    if (contentType === null) {
+        return null;
+    }
+    const normalized = contentType.trim().toLowerCase();
+    return normalized === '' ? null : normalized;
+}

package/lib/crawler/partition-urls-by-html.d.ts

@@ -0,0 +1,16 @@
+import type { ExURL } from '@d-zero/shared/parse-url';
+/**
+ * Split URLs into a likely-HTML group and a non-HTML group, preserving input
+ * order within each group.
+ *
+ * Used by the crawler's enqueue path to route discovered URLs: likely-HTML URLs
+ * are `unshift`ed to the front of the dealer queue (so page crawling advances
+ * ahead of asset/document fetches) while the rest are `push`ed to the tail.
+ * Keeping the order stable within each group means a batch (e.g. predicted
+ * pagination URLs) stays in ascending order at the front when unshifted as one
+ * call. Classification is delegated to {@link isLikelyHtmlUrl}.
+ * @param urls - The discovered URLs to partition.
+ * @returns A `[html, other]` tuple: likely-HTML URLs and the remainder, each in
+ *   original input order.
+ */
+export declare function partitionUrlsByHtml(urls: readonly ExURL[]): [html: ExURL[], other: ExURL[]];

package/lib/crawler/partition-urls-by-html.js

@@ -0,0 +1,23 @@
+import { isLikelyHtmlUrl } from './is-likely-html-url.js';
+/**
+ * Split URLs into a likely-HTML group and a non-HTML group, preserving input
+ * order within each group.
+ *
+ * Used by the crawler's enqueue path to route discovered URLs: likely-HTML URLs
+ * are `unshift`ed to the front of the dealer queue (so page crawling advances
+ * ahead of asset/document fetches) while the rest are `push`ed to the tail.
+ * Keeping the order stable within each group means a batch (e.g. predicted
+ * pagination URLs) stays in ascending order at the front when unshifted as one
+ * call. Classification is delegated to {@link isLikelyHtmlUrl}.
+ * @param urls - The discovered URLs to partition.
+ * @returns A `[html, other]` tuple: likely-HTML URLs and the remainder, each in
+ *   original input order.
+ */
+export function partitionUrlsByHtml(urls) {
+    const html = [];
+    const other = [];
+    for (const url of urls) {
+        (isLikelyHtmlUrl(url) ? html : other).push(url);
+    }
+    return [html, other];
+}

package/lib/crawler/redirect-dest-key.d.ts

@@ -0,0 +1,19 @@
+import type { ExURL } from '@d-zero/shared/parse-url';
+/**
+ * Computes the protocol-agnostic dedup key for the final destination a request
+ * lands on after following its redirect chain.
+ *
+ * Used by the redirect-convergence optimisation (#73): the crawler remembers
+ * which final destinations have already been rendered, keyed by this value, so
+ * that many source URLs all redirecting to one destination render it only once.
+ * When there is no redirect, the destination is the requested URL itself.
+ *
+ * The key matches the form used elsewhere in the crawler (`protocolAgnosticKey`
+ * over the normalised URL without hash/auth) so HTTP and HTTPS variants of the
+ * same destination collapse to one entry.
+ * @param url - The originally requested URL.
+ * @param redirectPaths - The redirect hop URLs captured during the HEAD
+ *   pre-flight, in order. The last entry is the final destination.
+ * @returns The protocol-agnostic key of the final destination.
+ */
+export declare function redirectDestKey(url: ExURL, redirectPaths: readonly string[]): string;

package/lib/crawler/redirect-dest-key.js

@@ -0,0 +1,27 @@
+import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
+import { protocolAgnosticKey } from './protocol-agnostic-key.js';
+/**
+ * Computes the protocol-agnostic dedup key for the final destination a request
+ * lands on after following its redirect chain.
+ *
+ * Used by the redirect-convergence optimisation (#73): the crawler remembers
+ * which final destinations have already been rendered, keyed by this value, so
+ * that many source URLs all redirecting to one destination render it only once.
+ * When there is no redirect, the destination is the requested URL itself.
+ *
+ * The key matches the form used elsewhere in the crawler (`protocolAgnosticKey`
+ * over the normalised URL without hash/auth) so HTTP and HTTPS variants of the
+ * same destination collapse to one entry.
+ * @param url - The originally requested URL.
+ * @param redirectPaths - The redirect hop URLs captured during the HEAD
+ *   pre-flight, in order. The last entry is the final destination.
+ * @returns The protocol-agnostic key of the final destination.
+ */
+export function redirectDestKey(url, redirectPaths) {
+    const last = redirectPaths.at(-1);
+    if (last === undefined) {
+        return protocolAgnosticKey(url.withoutHashAndAuth);
+    }
+    const parsed = parseUrl(last);
+    return protocolAgnosticKey(parsed ? parsed.withoutHashAndAuth : last);
+}

package/lib/crawler/resource-to-page-data.d.ts

@@ -0,0 +1,28 @@
+import type { ResourceLookupResult } from './types.js';
+import type { PageData } from '../utils/types/types.js';
+import type { ExURL } from '@d-zero/shared/parse-url';
+/**
+ * Parameters for {@link resourceToPageData}.
+ */
+interface ResourceToPageDataParams {
+    /** The queued URL being processed. */
+    readonly url: ExURL;
+    /** Whether the URL is external to the crawl scope. */
+    readonly isExternal: boolean;
+    /** The recorded sub-resource data captured during page rendering. */
+    readonly resource: ResourceLookupResult;
+}
+/**
+ * Synthesize {@link PageData} from a recorded sub-resource row, or return
+ * `null` when the resource is not eligible for reuse.
+ *
+ * Eligible: the status is 2xx AND the content type is known and not
+ * `text/html`. Non-2xx rows (redirect hops, errors, 304s) and HTML rows must
+ * fall back to the normal HEAD pre-flight / browser scrape, which also
+ * guarantees that `redirectPaths: []` here is accurate — a URL that
+ * redirects is recorded with its 3xx status and never reaches this path.
+ * @param params - The queued URL, its external flag, and the recorded resource data.
+ * @returns The synthesized page data, or `null` when the caller must fall back.
+ */
+export declare function resourceToPageData(params: ResourceToPageDataParams): PageData | null;
+export {};

package/lib/crawler/resource-to-page-data.js

@@ -0,0 +1,59 @@
+import { isHtmlContentType } from './is-html-content-type.js';
+/**
+ * Synthesize {@link PageData} from a recorded sub-resource row, or return
+ * `null` when the resource is not eligible for reuse.
+ *
+ * Eligible: the status is 2xx AND the content type is known and not
+ * `text/html`. Non-2xx rows (redirect hops, errors, 304s) and HTML rows must
+ * fall back to the normal HEAD pre-flight / browser scrape, which also
+ * guarantees that `redirectPaths: []` here is accurate — a URL that
+ * redirects is recorded with its 3xx status and never reaches this path.
+ * @param params - The queued URL, its external flag, and the recorded resource data.
+ * @returns The synthesized page data, or `null` when the caller must fall back.
+ */
+export function resourceToPageData(params) {
+    const { url, isExternal, resource } = params;
+    if (resource.status == null ||
+        resource.status < 200 ||
+        resource.status >= 300 ||
+        resource.contentType == null ||
+        isHtmlContentType(resource.contentType)) {
+        return null;
+    }
+    return {
+        url,
+        redirectPaths: [],
+        isTarget: !isExternal,
+        isExternal,
+        status: resource.status,
+        statusText: resource.statusText ?? '',
+        contentType: resource.contentType,
+        contentLength: resource.contentLength,
+        responseHeaders: resource.responseHeaders,
+        // beholder 3.0.0 made jsonLd / speculationRules / tags / others /
+        // originTrial required Meta fields. This fallback wraps a non-HTML
+        // resource (PDF, image, etc.) that was discovered via subresource
+        // inspection rather than full scraping; populate every required slot
+        // so downstream insert/derive helpers iterate without crashing.
+        meta: {
+            title: '',
+            jsonLd: [],
+            speculationRules: [],
+            tags: { detected: {}, entries: [] },
+            others: {
+                meta: {},
+                property: {},
+                httpEquiv: {},
+                itemprop: {},
+                link: [],
+                script: [],
+                iframe: [],
+            },
+            originTrial: [],
+        },
+        anchorList: [],
+        imageList: [],
+        html: '',
+        isSkipped: false,
+    };
+}