@nitpicker/crawler 0.9.0 → 0.12.0

package/lib/crawler/fetch-destination.js

@@ -2,22 +2,26 @@ import { delay } from '@d-zero/shared/delay';
 import redirects from 'follow-redirects';
 import { destinationCache } from './destination-cache.js';
 import NetTimeoutError from './net-timeout-error.js';
+import { shouldGetFallbackOnHeadFailure } from './should-get-fallback-on-head-failure.js';
+/** Default race timeout for the HEAD pre-flight, in milliseconds. */
+const DEFAULT_HEAD_TIMEOUT_MS = 10 * 1000;
 /**
  * Fetches the destination metadata for a URL using an HTTP HEAD request (or GET as fallback).
  *
  * Results are cached in memory so that repeated calls for the same URL
- * (without hash) return immediately. The request races against a 10-second
- * timeout; if the server does not respond in time, a {@link NetTimeoutError} is thrown.
+ * (without hash) return immediately. The request races against a configurable
+ * timeout (defaults to {@link DEFAULT_HEAD_TIMEOUT_MS}, 10 seconds); if the
+ * server does not respond in time, a {@link NetTimeoutError} is thrown.
  *
  * If the server returns 405 (Method Not Allowed), 501 (Not Implemented), or 503
  * (Service Unavailable) for a HEAD request, the function automatically retries with GET.
- * @param params - Parameters containing URL, external flag, method, options, and optional User-Agent.
+ * @param params - Parameters containing URL, external flag, method, options, optional User-Agent, and optional timeout.
  * @returns The page metadata obtained from the HTTP response.
- * @throws {NetTimeoutError} If the request exceeds the 10-second timeout.
+ * @throws {NetTimeoutError} If the request exceeds the configured timeout.
  * @throws {Error} If the HTTP request fails for any other reason.
  */
 export async function fetchDestination(params) {
-    const { url, isExternal, method = 'HEAD', options, userAgent } = params;
+    const { url, isExternal, method = 'HEAD', options, userAgent, timeout } = params;
     const titleBytesLimit = options?.titleBytesLimit;
     const cacheKey = titleBytesLimit == null ? url.withoutHash : `${url.withoutHash}:title`;
     if (destinationCache.has(cacheKey)) {
@@ -28,20 +32,65 @@ export async function fetchDestination(params) {
         return cache;
     }
     const effectiveMethod = titleBytesLimit == null ? method : 'GET';
-    // Race the fetch against a 10-second timeout. The losing timer is cleared
+    const raceTimeoutMs = timeout ?? DEFAULT_HEAD_TIMEOUT_MS;
+    // Race the fetch against the requested timeout. The losing timer is cleared
     // explicitly so it never keeps the event loop alive after the race settles
     // (a plain `delay()` in `Promise.race` would leak the timer until it fires).
     let timeoutHandle;
     const result = await Promise.race([
-        _fetchHead(url, isExternal, effectiveMethod, titleBytesLimit, userAgent).catch((error) => (error instanceof Error ? error : new Error(String(error)))),
+        _fetchHead(url, isExternal, effectiveMethod, titleBytesLimit, userAgent, timeout).catch((error) => error instanceof Error ? error : new Error(String(error))),
         new Promise((resolve) => {
-            timeoutHandle = setTimeout(() => resolve(new NetTimeoutError(url.href)), 10 * 1000);
+            timeoutHandle = setTimeout(() => resolve(new NetTimeoutError(url.href)), raceTimeoutMs);
         }),
     ]).finally(() => {
         if (timeoutHandle)
             clearTimeout(timeoutHandle);
     });
-    destinationCache.set(cacheKey, result);
+    // HEAD failure fallback: a WAF / middlebox that silently drops HEAD will
+    // surface as NetTimeoutError / parse-error / connection-reset here even
+    // though the same URL serves a normal GET response. Try GET once (using
+    // the same timeout budget) before giving up on the URL. Only when
+    // `method === 'HEAD'` to avoid infinite recursion if the GET itself
+    // times out — at that point the server really is unreachable.
+    if (method === 'HEAD' &&
+        result instanceof Error &&
+        shouldGetFallbackOnHeadFailure(result)) {
+        try {
+            const getResult = await fetchDestination({
+                url,
+                isExternal,
+                method: 'GET',
+                userAgent,
+                timeout,
+            });
+            // GET succeeded — that is the canonical answer for this URL, so
+            // cache it under the HEAD cacheKey too (same key, since cacheKey
+            // only depends on URL + titleBytesLimit, not on method). The
+            // inner GET call already wrote to the cache under the same key,
+            // but a future caller hitting the HEAD path will find it there.
+            return getResult;
+        }
+        catch {
+            // GET fallback failed too; fall through to surface the original
+            // HEAD failure so retry / classification / DNS-burned cache see
+            // the actual underlying cause.
+        }
+    }
+    // Errors that are RECOVERABLE on retry — NetTimeoutError plus the kinds
+    // `shouldGetFallbackOnHeadFailure` already singles out as
+    // possibly-recoverable (parse-error, connection-reset) — are
+    // intentionally NOT cached. Caching a recoverable failure would freeze
+    // the first slow probe as the verdict for every later caller on the
+    // same host AND defeat `Crawler.#sendHeadRequest`'s
+    // HEAD_TIMEOUT_ESCALATION_MS (the 30s/60s retry would hit the cache and
+    // re-throw the stale 10s failure instead of getting the longer
+    // budget). DNS / TLS / refused / blocked are persistent within a crawl
+    // session so caching them is what keeps a doomed host from re-paying
+    // the network cost N times.
+    const isRecoverableError = result instanceof Error && shouldGetFallbackOnHeadFailure(result);
+    if (!isRecoverableError) {
+        destinationCache.set(cacheKey, result);
+    }
     if (result instanceof Error) {
         throw result;
     }
@@ -58,17 +107,26 @@ export async function fetchDestination(params) {
  * @param titleBytesLimit - When set, reads up to this many bytes from the response body
  *   to extract a `<title>` tag, then destroys the connection.
  * @param userAgent - Optional User-Agent string to send with the request.
+ * @param timeout - Optional race timeout in ms, forwarded to GET fallback so the
+ *   second pass keeps the same budget as the original HEAD attempt.
  * @returns A promise resolving to {@link PageData} with response metadata.
  */
-async function _fetchHead(url, isExternal, method, titleBytesLimit, userAgent) {
+async function _fetchHead(url, isExternal, method, titleBytesLimit, userAgent, timeout) {
     return new Promise((resolve, reject) => {
         const hostHeader = url.port ? `${url.hostname}:${url.port}` : url.hostname;
+        // `trackRedirects` makes follow-redirects populate `res.redirects` with the
+        // chain of followed URLs. Without it that array stays empty and the
+        // pre-flight cannot tell where a URL lands — required for the redirect
+        // chain in `redirectPaths` and for the #73 convergence dedup, which decides
+        // whether a redirect destination was already rendered *before* launching
+        // the browser.
         const request = {
             protocol: url.protocol,
             hostname: url.hostname,
             port: url.port || undefined,
             path: url.pathname,
             method,
+            trackRedirects: true,
             headers: {
                 host: hostHeader,
                 ...(userAgent ? { 'User-Agent': userAgent } : {}),
@@ -92,7 +150,17 @@ async function _fetchHead(url, isExternal, method, titleBytesLimit, userAgent) {
             let totalBytes = 0;
             let settled = false;
             const buildPageData = (title) => {
-                const redirectPaths = res.redirects.map((r) => r.url);
+                // `res.redirects` (populated by trackRedirects) ALWAYS starts with the
+                // originally requested URL, then each followed hop. We drop that first
+                // entry so `redirectPaths` keeps its established contract: empty when the
+                // URL did not redirect, and `[...intermediate, finalDest]` when it did
+                // (the original URL is NOT included — callers like `resolveRedirectChain`
+                // and `updatePage` re-add it). Keeping the original here would (a) make
+                // `redirectPaths` non-empty for every page, so a direct page looks like a
+                // self-redirect, and (b) leak the query-stripped request-target (the HEAD
+                // request uses `url.pathname`), collapsing query-distinguished pages.
+                // Redirect *targets* come from Location headers and keep their query.
+                const redirectPaths = res.redirects.map((r) => r.url).slice(1);
                 const _contentLength = Number.parseInt(res.headers['content-length'] || '');
                 const contentLength = Number.isFinite(_contentLength) ? _contentLength : null;
                 return {
@@ -105,7 +173,26 @@ async function _fetchHead(url, isExternal, method, titleBytesLimit, userAgent) {
                     contentType: res.headers['content-type']?.split(';')[0] || null,
                     contentLength,
                     responseHeaders: res.headers,
-                    meta: { title },
+                    // beholder 3.0.0 made jsonLd / speculationRules / tags /
+                    // others / originTrial required Meta fields. Even this
+                    // HEAD-only fallback path must populate every slot so
+                    // downstream insert/derive helpers iterate without crashing.
+                    meta: {
+                        title,
+                        jsonLd: [],
+                        speculationRules: [],
+                        tags: { detected: {}, entries: [] },
+                        others: {
+                            meta: {},
+                            property: {},
+                            httpEquiv: {},
+                            itemprop: {},
+                            link: [],
+                            script: [],
+                            iframe: [],
+                        },
+                        originTrial: [],
+                    },
                     imageList: [],
                     anchorList: [],
                     html: '',
@@ -118,11 +205,21 @@ async function _fetchHead(url, isExternal, method, titleBytesLimit, userAgent) {
                     let rep = buildPageData('');
                     if (rep.status === 405) {
                         if (method === 'GET') {
-                            reject(new Error(`Method Not Allowed: ${url.href} ${rep.statusText}`));
+                            // GET fallback also returned 405 — the server really does
+                            // reject both methods. Resolve with the PageData so the
+                            // archive records `status: 405` instead of the `-1`
+                            // sentinel a reject would land on (which would erase the
+                            // only useful diagnostic the server gave us).
+                            resolve(rep);
                             return;
                         }
                         try {
-                            rep = await fetchDestination({ url, isExternal, method: 'GET' });
+                            rep = await fetchDestination({
+                                url,
+                                isExternal,
+                                method: 'GET',
+                                timeout,
+                            });
                         }
                         catch (error) {
                             reject(error);
@@ -131,12 +228,19 @@ async function _fetchHead(url, isExternal, method, titleBytesLimit, userAgent) {
                     }
                     if (rep.status === 501) {
                         if (method === 'GET') {
-                            reject(new Error(`Method Not Implemented: ${url.href} ${rep.statusText}`));
+                            // GET fallback also returned 501 — preserve the status
+                            // rather than dropping it into the `-1` bucket.
+                            resolve(rep);
                             return;
                         }
                         await delay(5 * 1000);
                         try {
-                            rep = await fetchDestination({ url, isExternal, method: 'GET' });
+                            rep = await fetchDestination({
+                                url,
+                                isExternal,
+                                method: 'GET',
+                                timeout,
+                            });
                         }
                         catch (error) {
                             reject(error);
@@ -145,12 +249,22 @@ async function _fetchHead(url, isExternal, method, titleBytesLimit, userAgent) {
                     }
                     if (rep.status === 503) {
                         if (method === 'GET') {
-                            reject(new Error(`Retrying failed: ${url.href} ${rep.statusText}`));
+                            // GET fallback also returned 503 — preserve the status.
+                            // A second-pass 5xx from a different method is the
+                            // server's real answer, not a transient HEAD-only quirk,
+                            // so the archive should remember it as 503 instead of
+                            // the generic `-1` sentinel.
+                            resolve(rep);
                             return;
                         }
                         await delay(5 * 1000);
                         try {
-                            rep = await fetchDestination({ url, isExternal, method: 'GET' });
+                            rep = await fetchDestination({
+                                url,
+                                isExternal,
+                                method: 'GET',
+                                timeout,
+                            });
                         }
                         catch (error) {
                             reject(error);

package/lib/crawler/format-crawl-progress.d.ts

@@ -12,22 +12,31 @@ interface FormatCrawlProgressParams {
     readonly externalTotal: number;
     /** Number of external URLs completed */
     readonly externalDone: number;
+    /**
+     * Number of HTML pages actually rendered by the browser in the current session.
+     * Excludes HEAD-only resources, title-only metadata fetches, and skipped URLs.
+     */
+    readonly pagesScraped: number;
     /** Number of parallel workers */
     readonly limit: number;
 }
 /**
  * Formats the crawl progress header for the deal() progress display.
  *
- * Shows "done / found (remaining)" format instead of "done/total"
- * to make it clearer that the total is expected to grow during crawling.
+ * Shows "done(pages) / found URLs (remaining)" format instead of "done/total"
+ * to make it clearer that the total is expected to grow during crawling,
+ * that the counts are processed URLs (not resulting pages), and how many of
+ * those URLs were actually rendered by the browser as HTML pages.
+ * Counts are formatted with thousands separators (e.g. `1,234,567`).
  * @param params - The crawl progress parameters.
  * @param params.done - Number of URLs completed by the deal queue.
  * @param params.total - Total number of URLs in the deal queue (including completed).
  * @param params.resumeOffset - Offset from a previous resumed session.
  * @param params.externalTotal - Number of external URLs discovered.
  * @param params.externalDone - Number of external URLs completed.
+ * @param params.pagesScraped - Number of HTML pages rendered by the browser in this session.
  * @param params.limit - Number of parallel workers.
  * @returns The formatted progress string with ANSI color codes.
  */
-export declare function formatCrawlProgress({ done, total, resumeOffset, externalTotal, externalDone, limit, }: FormatCrawlProgressParams): string;
+export declare function formatCrawlProgress({ done, total, resumeOffset, externalTotal, externalDone, pagesScraped, limit, }: FormatCrawlProgressParams): string;
 export {};

package/lib/crawler/format-crawl-progress.js

@@ -1,19 +1,27 @@
 import c from 'ansi-colors';
+/**
+ * Number formatter for thousands-separated count display (e.g. `1,234,567`).
+ */
+const countFormat = new Intl.NumberFormat('en-US');
 /**
  * Formats the crawl progress header for the deal() progress display.
  *
- * Shows "done / found (remaining)" format instead of "done/total"
- * to make it clearer that the total is expected to grow during crawling.
+ * Shows "done(pages) / found URLs (remaining)" format instead of "done/total"
+ * to make it clearer that the total is expected to grow during crawling,
+ * that the counts are processed URLs (not resulting pages), and how many of
+ * those URLs were actually rendered by the browser as HTML pages.
+ * Counts are formatted with thousands separators (e.g. `1,234,567`).
  * @param params - The crawl progress parameters.
  * @param params.done - Number of URLs completed by the deal queue.
  * @param params.total - Total number of URLs in the deal queue (including completed).
  * @param params.resumeOffset - Offset from a previous resumed session.
  * @param params.externalTotal - Number of external URLs discovered.
  * @param params.externalDone - Number of external URLs completed.
+ * @param params.pagesScraped - Number of HTML pages rendered by the browser in this session.
  * @param params.limit - Number of parallel workers.
  * @returns The formatted progress string with ANSI color codes.
  */
-export function formatCrawlProgress({ done, total, resumeOffset, externalTotal, externalDone, limit, }) {
+export function formatCrawlProgress({ done, total, resumeOffset, externalTotal, externalDone, pagesScraped, limit, }) {
     const allDone = done + resumeOffset;
     const allTotal = total + resumeOffset;
     const internalDone = allDone - externalDone;
@@ -22,8 +30,8 @@ export function formatCrawlProgress({ done, total, resumeOffset, externalTotal,
     const externalRemaining = externalTotal - externalDone;
     const totalRemaining = internalRemaining + externalRemaining;
     const pct = allTotal > 0 ? Math.round((allDone / allTotal) * 100) : 0;
-    return (c.bold(`Crawling: ${internalDone} done / ${internalTotal} found`) +
-        c.dim(` (+${externalDone}/${externalTotal} ext)`) +
-        c.bold(` (${pct}%) [${totalRemaining} remaining]`) +
+    return (c.bold(`Crawling: ${countFormat.format(internalDone)}(${countFormat.format(pagesScraped)}) done / ${countFormat.format(internalTotal)} found URLs`) +
+        c.dim(` (+${countFormat.format(externalDone)}/${countFormat.format(externalTotal)} ext)`) +
+        c.bold(` (${pct}%) [${countFormat.format(totalRemaining)} remaining]`) +
         c.dim(` [${limit} parallel]`));
 }

package/lib/crawler/handle-browser-close.d.ts

@@ -0,0 +1,29 @@
+import type { ClosableBrowser } from './close-browser-safely.js';
+/**
+ * Debug-style logger compatible with the `debug` package's printf-style API.
+ *
+ * Declared structurally so the function stays unit-testable with a `vi.fn()`
+ * stub and free of a `debug` import.
+ */
+export type BrowserCloseLogger = (
+/** printf-style format string (e.g. `'%s'`, `'%O'`). */
+formatter: string, 
+/** Arguments interpolated into the format string. */
+...args: readonly unknown[]) => void;
+/**
+ * Closes a Puppeteer browser used to scrape a single URL, recording any
+ * timeout fallback or unexpected cleanup error to the supplied logger.
+ *
+ * WHY a dedicated function: the prior inline `finally` in
+ * {@link Crawler.#launchBrowserAndScrape} mixed cleanup orchestration with
+ * production log formatting. Splitting it out makes the two observable
+ * branches (force-kill notice and unexpected-error notice) directly testable
+ * without spawning a real browser, and keeps the rule that a finally block
+ * never throws: any error from {@link closeBrowserSafely} is logged here,
+ * never re-thrown.
+ * @param browser - The browser to close.
+ * @param urlHref - URL string included in the log messages for diagnostic
+ *   context.
+ * @param log - Logger used to record timeout and error events.
+ */
+export declare function handleBrowserClose(browser: ClosableBrowser, urlHref: string, log: BrowserCloseLogger): Promise<void>;

package/lib/crawler/handle-browser-close.js

@@ -0,0 +1,28 @@
+import { closeBrowserSafely } from './close-browser-safely.js';
+/**
+ * Closes a Puppeteer browser used to scrape a single URL, recording any
+ * timeout fallback or unexpected cleanup error to the supplied logger.
+ *
+ * WHY a dedicated function: the prior inline `finally` in
+ * {@link Crawler.#launchBrowserAndScrape} mixed cleanup orchestration with
+ * production log formatting. Splitting it out makes the two observable
+ * branches (force-kill notice and unexpected-error notice) directly testable
+ * without spawning a real browser, and keeps the rule that a finally block
+ * never throws: any error from {@link closeBrowserSafely} is logged here,
+ * never re-thrown.
+ * @param browser - The browser to close.
+ * @param urlHref - URL string included in the log messages for diagnostic
+ *   context.
+ * @param log - Logger used to record timeout and error events.
+ */
+export async function handleBrowserClose(browser, urlHref, log) {
+    try {
+        const timedOut = await closeBrowserSafely(browser);
+        if (timedOut) {
+            log('Force-killed wedged Chromium browser for %s (close() timed out)', urlHref);
+        }
+    }
+    catch (error) {
+        log('closeBrowserSafely failed for %s: %O', urlHref, error);
+    }
+}

package/lib/crawler/is-html-content-type.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Determine whether a Content-Type media type is HTML.
+ *
+ * MIME types are case-insensitive (RFC 2045), and values captured from
+ * Puppeteer responses preserve the server's original casing, so the
+ * comparison must normalize case — `text/HTML` is HTML. Surrounding
+ * whitespace (e.g. `text/html ` left over after parameter stripping)
+ * is also tolerated.
+ *
+ * This is the single source of truth for HTML detection — `Page.isPage()`
+ * and the link list delegate here so the classification never diverges
+ * between code paths.
+ * @param contentType - The media type portion of a Content-Type header
+ *   (parameters already stripped), or `null` when unknown.
+ * @returns `true` when the media type is `text/html` in any letter case.
+ */
+export declare function isHtmlContentType(contentType: string | null): boolean;

package/lib/crawler/is-html-content-type.js

@@ -0,0 +1,19 @@
+/**
+ * Determine whether a Content-Type media type is HTML.
+ *
+ * MIME types are case-insensitive (RFC 2045), and values captured from
+ * Puppeteer responses preserve the server's original casing, so the
+ * comparison must normalize case — `text/HTML` is HTML. Surrounding
+ * whitespace (e.g. `text/html ` left over after parameter stripping)
+ * is also tolerated.
+ *
+ * This is the single source of truth for HTML detection — `Page.isPage()`
+ * and the link list delegate here so the classification never diverges
+ * between code paths.
+ * @param contentType - The media type portion of a Content-Type header
+ *   (parameters already stripped), or `null` when unknown.
+ * @returns `true` when the media type is `text/html` in any letter case.
+ */
+export function isHtmlContentType(contentType) {
+    return contentType !== null && contentType.trim().toLowerCase() === 'text/html';
+}

package/lib/crawler/is-js-redirect-error-shape.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Decide whether a browser-scrape error message is the specific
+ * `Page.goto() returned null` shape that the JS-redirect rescue is
+ * designed to recover from.
+ *
+ * **Why this gate exists:** before the gate, the rescue fired on *any*
+ * thrown error from `scraper.scrapeStart` as long as `page.url()` happened
+ * to report a different http(s) URL. That made every browser failure
+ * (TLS, target-crashed, OOM, navigation timeout, …) that incidentally
+ * left the page on a follow-up URL look like a JS redirect, hiding the
+ * real failure mode and stamping a phantom `status = 301` on the source.
+ *
+ * The narrow trigger only fires on the upstream's exact sentinel —
+ * `Page.goto returned null` — which beholder's scraper throws *only*
+ * when puppeteer's `page.goto()` resolved to `null`. Substring match (not
+ * equality) so wrapped variants like `[Retried 3 times] The method
+ * Page.goto returned null` (which surface in `crawl_errors` after retry
+ * exhaustion at outer layers) still classify, even though the rescue
+ * sees the bare form. Case-insensitive on the marker so a future
+ * beholder bump that lowercases the message keeps working.
+ *
+ * The trigger keys off the message *string*, not the message-classifier
+ * `kind`, because the rescue runs *before* the kind decision: the kind
+ * classifier would already wash this into `protocol`, and `protocol`
+ * covers more than just goto-null (Target closed / Session closed /
+ * detached Frame …) — none of which leave puppeteer with a meaningful
+ * post-navigation URL to recover.
+ * @param message - The raw error message from
+ *   `BrowserScrapeResult.error.message` (or any string that may carry
+ *   the sentinel inside a wrapper). `null` / `undefined` returns `false`.
+ * @returns `true` iff the message carries the `Page.goto returned null`
+ *   sentinel.
+ * @example
+ * ```ts
+ * isJsRedirectErrorShape('The method Page.goto returned null'); // → true
+ * isJsRedirectErrorShape('Navigation timeout of 60000 ms exceeded'); // → false
+ * isJsRedirectErrorShape(undefined); // → false
+ * ```
+ */
+export declare function isJsRedirectErrorShape(message: string | null | undefined): boolean;

package/lib/crawler/is-js-redirect-error-shape.js

@@ -0,0 +1,53 @@
+/**
+ * Sentinel string emitted by `@d-zero/beholder`'s scraper when
+ * `await page.goto(...)` resolves to `null`. Pinned here because the
+ * JS-redirect rescue in `Crawler.#scrapePage` keys off the exact text —
+ * an upstream rename would silently disable the rescue, but the
+ * spec on this helper would also break, surfacing the drift in CI.
+ */
+const PAGE_GOTO_NULL_MARKER = 'Page.goto returned null';
+/**
+ * Decide whether a browser-scrape error message is the specific
+ * `Page.goto() returned null` shape that the JS-redirect rescue is
+ * designed to recover from.
+ *
+ * **Why this gate exists:** before the gate, the rescue fired on *any*
+ * thrown error from `scraper.scrapeStart` as long as `page.url()` happened
+ * to report a different http(s) URL. That made every browser failure
+ * (TLS, target-crashed, OOM, navigation timeout, …) that incidentally
+ * left the page on a follow-up URL look like a JS redirect, hiding the
+ * real failure mode and stamping a phantom `status = 301` on the source.
+ *
+ * The narrow trigger only fires on the upstream's exact sentinel —
+ * `Page.goto returned null` — which beholder's scraper throws *only*
+ * when puppeteer's `page.goto()` resolved to `null`. Substring match (not
+ * equality) so wrapped variants like `[Retried 3 times] The method
+ * Page.goto returned null` (which surface in `crawl_errors` after retry
+ * exhaustion at outer layers) still classify, even though the rescue
+ * sees the bare form. Case-insensitive on the marker so a future
+ * beholder bump that lowercases the message keeps working.
+ *
+ * The trigger keys off the message *string*, not the message-classifier
+ * `kind`, because the rescue runs *before* the kind decision: the kind
+ * classifier would already wash this into `protocol`, and `protocol`
+ * covers more than just goto-null (Target closed / Session closed /
+ * detached Frame …) — none of which leave puppeteer with a meaningful
+ * post-navigation URL to recover.
+ * @param message - The raw error message from
+ *   `BrowserScrapeResult.error.message` (or any string that may carry
+ *   the sentinel inside a wrapper). `null` / `undefined` returns `false`.
+ * @returns `true` iff the message carries the `Page.goto returned null`
+ *   sentinel.
+ * @example
+ * ```ts
+ * isJsRedirectErrorShape('The method Page.goto returned null'); // → true
+ * isJsRedirectErrorShape('Navigation timeout of 60000 ms exceeded'); // → false
+ * isJsRedirectErrorShape(undefined); // → false
+ * ```
+ */
+export function isJsRedirectErrorShape(message) {
+    if (typeof message !== 'string' || message === '') {
+        return false;
+    }
+    return message.toLowerCase().includes(PAGE_GOTO_NULL_MARKER.toLowerCase());
+}

package/lib/crawler/is-likely-html-url.d.ts

@@ -0,0 +1,22 @@
+import type { ExURL } from '@d-zero/shared/parse-url';
+/**
+ * Heuristically decide whether a discovered URL is likely to resolve to an
+ * HTML page, based solely on the URL itself.
+ *
+ * WHY URL-only: this runs at enqueue time — before any HEAD/GET — so the actual
+ * `Content-Type` is unknown. The crawler uses the result to prioritise the
+ * dealer queue (likely-HTML URLs are `unshift`ed to the front so page crawling
+ * advances ahead of asset/document fetches), so a heuristic is acceptable: a
+ * misclassification only changes fetch order, never correctness.
+ *
+ * Classification rules:
+ * - Non-HTTP URLs (`mailto:`, `tel:`, …) are never HTML pages.
+ * - Extensionless / directory-style URLs (`/`, `/about/`), and bare trailing-dot
+ *   URLs (`/index.`, whose `extname` is `"."`), are treated as HTML — these are
+ *   the overwhelmingly common shape for navigable pages.
+ * - URLs whose extension is in {@link HTML_EXTENSIONS} are HTML; every other
+ *   extension (`.jpg`, `.pdf`, `.css`, `.js`, …) is treated as non-HTML.
+ * @param url - The parsed URL to classify.
+ * @returns `true` when the URL is likely an HTML page.
+ */
+export declare function isLikelyHtmlUrl(url: ExURL): boolean;

package/lib/crawler/is-likely-html-url.js

@@ -0,0 +1,65 @@
+/**
+ * File extensions (lowercased, leading dot included) that typically denote a
+ * document served as `text/html`. Both static pages (`.html`) and the common
+ * server-side template / handler extensions (`.php`, `.aspx`, `.jsp`, `.ashx`,
+ * `.jsf` …) are listed because they usually serve an HTML page.
+ *
+ * A few entries (`.cgi`, `.do`, `.action`) are ambiguous — they sometimes
+ * return JSON or binary — but are kept here because a misclassification only
+ * changes fetch order, never correctness (see {@link isLikelyHtmlUrl}).
+ *
+ * Keys keep the leading dot so they can be compared directly against
+ * `ExURL.extname` (Node `Path.extname` output) without stripping it.
+ */
+const HTML_EXTENSIONS = new Set([
+    '.html',
+    '.htm',
+    '.xhtml',
+    '.shtml',
+    '.mhtml',
+    '.php',
+    '.php3',
+    '.php4',
+    '.php5',
+    '.phtml',
+    '.asp',
+    '.aspx',
+    '.ashx',
+    '.jsp',
+    '.jspx',
+    '.jsf',
+    '.cfm',
+    '.cgi',
+    '.do',
+    '.action',
+]);
+/**
+ * Heuristically decide whether a discovered URL is likely to resolve to an
+ * HTML page, based solely on the URL itself.
+ *
+ * WHY URL-only: this runs at enqueue time — before any HEAD/GET — so the actual
+ * `Content-Type` is unknown. The crawler uses the result to prioritise the
+ * dealer queue (likely-HTML URLs are `unshift`ed to the front so page crawling
+ * advances ahead of asset/document fetches), so a heuristic is acceptable: a
+ * misclassification only changes fetch order, never correctness.
+ *
+ * Classification rules:
+ * - Non-HTTP URLs (`mailto:`, `tel:`, …) are never HTML pages.
+ * - Extensionless / directory-style URLs (`/`, `/about/`), and bare trailing-dot
+ *   URLs (`/index.`, whose `extname` is `"."`), are treated as HTML — these are
+ *   the overwhelmingly common shape for navigable pages.
+ * - URLs whose extension is in {@link HTML_EXTENSIONS} are HTML; every other
+ *   extension (`.jpg`, `.pdf`, `.css`, `.js`, …) is treated as non-HTML.
+ * @param url - The parsed URL to classify.
+ * @returns `true` when the URL is likely an HTML page.
+ */
+export function isLikelyHtmlUrl(url) {
+    if (!url.isHTTP) {
+        return false;
+    }
+    const extname = url.extname;
+    if (!extname || extname === '.') {
+        return true;
+    }
+    return HTML_EXTENSIONS.has(extname.toLowerCase());
+}

package/lib/crawler/is-puppeteer-fallback-candidate.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Decide whether a failed HEAD/GET pre-flight error message warrants one
+ * puppeteer fallback attempt. Pure and deterministic — the same message
+ * always gives the same answer, so the decision can be unit-tested without
+ * spinning up a browser.
+ * @param message - The pre-flight error message (typically the last rejected
+ *   `retryCall` attempt).
+ * @returns `true` when puppeteer should be tried once, `false` to give up.
+ * @example
+ * ```ts
+ * isPuppeteerFallbackCandidate('Timeout: https://slow.example.org/'); // true
+ * isPuppeteerFallbackCandidate('getaddrinfo ENOTFOUND host.invalid'); // false
+ * isPuppeteerFallbackCandidate('net::ERR_CERT_DATE_INVALID');         // false
+ * ```
+ */
+export declare function isPuppeteerFallbackCandidate(message: string): boolean;