@nitpicker/crawler 0.9.0 → 0.11.0

package/lib/crawler/create-change-phase-handler.js

@@ -0,0 +1,44 @@
+/**
+ * Builds the `scraper.on('changePhase', ...)` listener used by
+ * {@link Crawler.#launchBrowserAndScrape}.
+ *
+ * Three responsibilities:
+ * 1. Render the phase log via the injected `formatLog` and pipe it to `update`.
+ * 2. Forward the raw event so external consumers (CLI progress UI etc.) see
+ *    every transition.
+ * 3. Buffer `retryExhausted` events into the per-URL phase-error map so they
+ *    can be drained as `pageError` events AFTER the `page` event fires.
+ *
+ * WHY a factory: the listener captures per-scrape state (`buffer`, `urlHref`,
+ * `update`). Extracting the factory makes the wiring directly unit-testable
+ * with plain stubs, instead of requiring a mocked Puppeteer + beholder + dealer
+ * stack to drive the worker.
+ *
+ * **Caller contract**: register the returned handler at most once per
+ * `scraper` instance. The Crawler creates a fresh Scraper per URL so this
+ * holds today; if scraper pooling is ever introduced, register exactly one
+ * handler per scrape and unregister it on completion to avoid duplicate
+ * buffer entries.
+ * @param options - Wiring dependencies for the handler.
+ * @returns A function suitable for `scraper.on('changePhase', ...)`.
+ */
+export function createChangePhaseHandler(options) {
+    const { emit, update, formatLog, buffer, urlHref } = options;
+    return (event) => {
+        const msg = formatLog(event);
+        if (msg) {
+            update(msg);
+        }
+        emit(event);
+        // retryExhausted fires when beholder's @retryable gives up on a
+        // secondary scrape step (e.g. a viewport switch detaching the frame
+        // in #fetchImages). The page itself still completes, so we buffer
+        // the failure here and emit it as a pageError after the page event
+        // has been emitted.
+        if (event.name === 'retryExhausted') {
+            const list = buffer.get(urlHref) ?? [];
+            list.push({ phase: event.name, message: event.message });
+            buffer.set(urlHref, list);
+        }
+    };
+}

package/lib/crawler/derive-page-source.d.ts

@@ -0,0 +1,23 @@
+import type { InventoryMode } from './types.js';
+import type { PageSource } from '../archive/types.js';
+/**
+ * Decide which {@link PageSource} label a newly-scraped page row should carry.
+ *
+ * When the crawler is NOT in inventory mode (`inventoryMode === null`),
+ * returns `undefined` — the caller emits no `source` and the DB DEFAULT
+ * `'crawled'` ends up on the row. This keeps the normal crawl path
+ * untouched.
+ *
+ * When inventory mode is active, the URL is matched against
+ * `inventoryMode.seedUrls`. A hit means the URL came straight from the
+ * user-supplied list (`'inventory-seed'`); a miss means the URL was found
+ * by following links from a seed page (`'inventory-discovered'`).
+ *
+ * Sub-resources captured by puppeteer during inventory-mode rendering are
+ * NEVER seeds — the caller for those events always passes
+ * `'inventory-discovered'` directly without consulting this helper.
+ * @param inventoryMode - Inventory-mode config from `CrawlerOptions.inventoryMode`, or `null` outside `--inventory`.
+ * @param pageUrlWithoutHashAndAuth - The page URL keyed by `withoutHashAndAuth` (auth credentials stripped, hash dropped).
+ * @returns The label to write to `pages.source`, or `undefined` for the DB default.
+ */
+export declare function derivePageSource(inventoryMode: InventoryMode | null, pageUrlWithoutHashAndAuth: string): PageSource | undefined;

package/lib/crawler/derive-page-source.js

@@ -0,0 +1,28 @@
+/**
+ * Decide which {@link PageSource} label a newly-scraped page row should carry.
+ *
+ * When the crawler is NOT in inventory mode (`inventoryMode === null`),
+ * returns `undefined` — the caller emits no `source` and the DB DEFAULT
+ * `'crawled'` ends up on the row. This keeps the normal crawl path
+ * untouched.
+ *
+ * When inventory mode is active, the URL is matched against
+ * `inventoryMode.seedUrls`. A hit means the URL came straight from the
+ * user-supplied list (`'inventory-seed'`); a miss means the URL was found
+ * by following links from a seed page (`'inventory-discovered'`).
+ *
+ * Sub-resources captured by puppeteer during inventory-mode rendering are
+ * NEVER seeds — the caller for those events always passes
+ * `'inventory-discovered'` directly without consulting this helper.
+ * @param inventoryMode - Inventory-mode config from `CrawlerOptions.inventoryMode`, or `null` outside `--inventory`.
+ * @param pageUrlWithoutHashAndAuth - The page URL keyed by `withoutHashAndAuth` (auth credentials stripped, hash dropped).
+ * @returns The label to write to `pages.source`, or `undefined` for the DB default.
+ */
+export function derivePageSource(inventoryMode, pageUrlWithoutHashAndAuth) {
+    if (inventoryMode === null) {
+        return undefined;
+    }
+    return inventoryMode.seedUrls.has(pageUrlWithoutHashAndAuth)
+        ? 'inventory-seed'
+        : 'inventory-discovered';
+}

package/lib/crawler/derive-resource-source.d.ts

@@ -0,0 +1,23 @@
+import type { InventoryMode } from './types.js';
+import type { PageSource } from '../archive/types.js';
+/**
+ * Decide which {@link PageSource} label a newly-captured sub-resource row
+ * (CSS / JS / image / fetch response) should carry.
+ *
+ * Sub-resources are NEVER themselves seeds — even when puppeteer is
+ * rendering a page that *is* an inventory seed, the assets it pulls in
+ * are downstream and must be labelled `'inventory-discovered'`. The seed
+ * label is reserved for URLs that were explicitly handed in by the user
+ * via the `--inventory` file.
+ *
+ * Outside inventory mode (`inventoryMode === null`) this returns
+ * `undefined` so the caller emits no `source` and the DB DEFAULT
+ * (`'crawled'`) lands on the row — keeps the normal crawl path
+ * untouched. This is the sub-resource counterpart of
+ * {@link import('./derive-page-source.js').derivePageSource}; the two
+ * helpers exist as a pair so a future addition to {@link PageSource}
+ * forces a parallel update.
+ * @param inventoryMode - Inventory-mode config from `CrawlerOptions.inventoryMode`, or `null` outside `--inventory`.
+ * @returns The label to write to `resources.source`, or `undefined` for the DB default.
+ */
+export declare function deriveResourceSource(inventoryMode: InventoryMode | null): PageSource | undefined;

package/lib/crawler/derive-resource-source.js

@@ -0,0 +1,26 @@
+/**
+ * Decide which {@link PageSource} label a newly-captured sub-resource row
+ * (CSS / JS / image / fetch response) should carry.
+ *
+ * Sub-resources are NEVER themselves seeds — even when puppeteer is
+ * rendering a page that *is* an inventory seed, the assets it pulls in
+ * are downstream and must be labelled `'inventory-discovered'`. The seed
+ * label is reserved for URLs that were explicitly handed in by the user
+ * via the `--inventory` file.
+ *
+ * Outside inventory mode (`inventoryMode === null`) this returns
+ * `undefined` so the caller emits no `source` and the DB DEFAULT
+ * (`'crawled'`) lands on the row — keeps the normal crawl path
+ * untouched. This is the sub-resource counterpart of
+ * {@link import('./derive-page-source.js').derivePageSource}; the two
+ * helpers exist as a pair so a future addition to {@link PageSource}
+ * forces a parallel update.
+ * @param inventoryMode - Inventory-mode config from `CrawlerOptions.inventoryMode`, or `null` outside `--inventory`.
+ * @returns The label to write to `resources.source`, or `undefined` for the DB default.
+ */
+export function deriveResourceSource(inventoryMode) {
+    if (inventoryMode === null) {
+        return undefined;
+    }
+    return 'inventory-discovered';
+}

package/lib/crawler/drain-phase-errors.d.ts

@@ -0,0 +1,48 @@
+/**
+ * A buffered phase-error record awaiting emission as a `pageError` event.
+ *
+ * `phase` is the beholder phase name (typically `'retryExhausted'`) and
+ * `message` is the human-readable failure text.
+ */
+export interface BufferedPhaseError {
+    /** Scrape phase name. */
+    phase: string;
+    /** Human-readable failure message. */
+    message: string;
+}
+/**
+ * Emitter signature accepted by {@link drainPhaseErrors} for the `pageError`
+ * event. Declared structurally so the Crawler's typed event emitter can be
+ * adapted with a thin closure at the call site without leaking through here.
+ */
+export type DrainPhaseErrorsEmit = (payload: {
+    /** URL of the affected page. */
+    url: string;
+    /** Scrape phase name. */
+    phase: string;
+    /** Human-readable failure message. */
+    message: string;
+    /** Whether the URL is external to the crawl scope. */
+    isExternal: boolean;
+}) => void;
+/**
+ * Drains the buffered phase errors for a URL: removes the entry from
+ * `buffer` and invokes `emit` once per buffered record.
+ *
+ * WHY a standalone function: the Crawler buffers `retryExhausted` events
+ * keyed by `url.href` during scrapeStart, then flushes them as
+ * `pageError` events AFTER `page` / `externalPage` has been emitted so the
+ * orchestrator's WriteQueue serialises `setPage` before `insertPageError`.
+ * Extracting the drain step here makes the flush + delete contract
+ * directly unit-testable without spinning up a real Crawler.
+ *
+ * Idempotent: calling twice for the same `urlHref` is safe — the second
+ * call sees an empty buffer and is a no-op.
+ * @param buffer - The pending-phase-errors map, keyed by URL href.
+ * @param urlHref - URL whose buffered errors should be drained.
+ * @param isExternal - Whether the URL is external to the crawl scope.
+ * @param emit - Callback invoked once per buffered phase-error record.
+ * @returns The number of phase-error events emitted (0 when the buffer
+ *   had no entry for `urlHref`).
+ */
+export declare function drainPhaseErrors(buffer: Map<string, BufferedPhaseError[]>, urlHref: string, isExternal: boolean, emit: DrainPhaseErrorsEmit): number;

package/lib/crawler/drain-phase-errors.js

@@ -0,0 +1,35 @@
+/**
+ * Drains the buffered phase errors for a URL: removes the entry from
+ * `buffer` and invokes `emit` once per buffered record.
+ *
+ * WHY a standalone function: the Crawler buffers `retryExhausted` events
+ * keyed by `url.href` during scrapeStart, then flushes them as
+ * `pageError` events AFTER `page` / `externalPage` has been emitted so the
+ * orchestrator's WriteQueue serialises `setPage` before `insertPageError`.
+ * Extracting the drain step here makes the flush + delete contract
+ * directly unit-testable without spinning up a real Crawler.
+ *
+ * Idempotent: calling twice for the same `urlHref` is safe — the second
+ * call sees an empty buffer and is a no-op.
+ * @param buffer - The pending-phase-errors map, keyed by URL href.
+ * @param urlHref - URL whose buffered errors should be drained.
+ * @param isExternal - Whether the URL is external to the crawl scope.
+ * @param emit - Callback invoked once per buffered phase-error record.
+ * @returns The number of phase-error events emitted (0 when the buffer
+ *   had no entry for `urlHref`).
+ */
+export function drainPhaseErrors(buffer, urlHref, isExternal, emit) {
+    const errors = buffer.get(urlHref);
+    if (!errors || errors.length === 0)
+        return 0;
+    buffer.delete(urlHref);
+    for (const err of errors) {
+        emit({
+            url: urlHref,
+            phase: err.phase,
+            message: err.message,
+            isExternal,
+        });
+    }
+    return errors.length;
+}

package/lib/crawler/fetch-destination.js

@@ -63,12 +63,19 @@ export async function fetchDestination(params) {
 async function _fetchHead(url, isExternal, method, titleBytesLimit, userAgent) {
     return new Promise((resolve, reject) => {
         const hostHeader = url.port ? `${url.hostname}:${url.port}` : url.hostname;
+        // `trackRedirects` makes follow-redirects populate `res.redirects` with the
+        // chain of followed URLs. Without it that array stays empty and the
+        // pre-flight cannot tell where a URL lands — required for the redirect
+        // chain in `redirectPaths` and for the #73 convergence dedup, which decides
+        // whether a redirect destination was already rendered *before* launching
+        // the browser.
         const request = {
             protocol: url.protocol,
             hostname: url.hostname,
             port: url.port || undefined,
             path: url.pathname,
             method,
+            trackRedirects: true,
             headers: {
                 host: hostHeader,
                 ...(userAgent ? { 'User-Agent': userAgent } : {}),
@@ -92,7 +99,17 @@ async function _fetchHead(url, isExternal, method, titleBytesLimit, userAgent) {
             let totalBytes = 0;
             let settled = false;
             const buildPageData = (title) => {
-                const redirectPaths = res.redirects.map((r) => r.url);
+                // `res.redirects` (populated by trackRedirects) ALWAYS starts with the
+                // originally requested URL, then each followed hop. We drop that first
+                // entry so `redirectPaths` keeps its established contract: empty when the
+                // URL did not redirect, and `[...intermediate, finalDest]` when it did
+                // (the original URL is NOT included — callers like `resolveRedirectChain`
+                // and `updatePage` re-add it). Keeping the original here would (a) make
+                // `redirectPaths` non-empty for every page, so a direct page looks like a
+                // self-redirect, and (b) leak the query-stripped request-target (the HEAD
+                // request uses `url.pathname`), collapsing query-distinguished pages.
+                // Redirect *targets* come from Location headers and keep their query.
+                const redirectPaths = res.redirects.map((r) => r.url).slice(1);
                 const _contentLength = Number.parseInt(res.headers['content-length'] || '');
                 const contentLength = Number.isFinite(_contentLength) ? _contentLength : null;
                 return {
@@ -105,7 +122,26 @@ async function _fetchHead(url, isExternal, method, titleBytesLimit, userAgent) {
                     contentType: res.headers['content-type']?.split(';')[0] || null,
                     contentLength,
                     responseHeaders: res.headers,
-                    meta: { title },
+                    // beholder 3.0.0 made jsonLd / speculationRules / tags /
+                    // others / originTrial required Meta fields. Even this
+                    // HEAD-only fallback path must populate every slot so
+                    // downstream insert/derive helpers iterate without crashing.
+                    meta: {
+                        title,
+                        jsonLd: [],
+                        speculationRules: [],
+                        tags: { detected: {}, entries: [] },
+                        others: {
+                            meta: {},
+                            property: {},
+                            httpEquiv: {},
+                            itemprop: {},
+                            link: [],
+                            script: [],
+                            iframe: [],
+                        },
+                        originTrial: [],
+                    },
                     imageList: [],
                     anchorList: [],
                     html: '',

package/lib/crawler/format-crawl-progress.d.ts

@@ -12,22 +12,31 @@ interface FormatCrawlProgressParams {
     readonly externalTotal: number;
     /** Number of external URLs completed */
     readonly externalDone: number;
+    /**
+     * Number of HTML pages actually rendered by the browser in the current session.
+     * Excludes HEAD-only resources, title-only metadata fetches, and skipped URLs.
+     */
+    readonly pagesScraped: number;
     /** Number of parallel workers */
     readonly limit: number;
 }
 /**
  * Formats the crawl progress header for the deal() progress display.
  *
- * Shows "done / found (remaining)" format instead of "done/total"
- * to make it clearer that the total is expected to grow during crawling.
+ * Shows "done(pages) / found URLs (remaining)" format instead of "done/total"
+ * to make it clearer that the total is expected to grow during crawling,
+ * that the counts are processed URLs (not resulting pages), and how many of
+ * those URLs were actually rendered by the browser as HTML pages.
+ * Counts are formatted with thousands separators (e.g. `1,234,567`).
  * @param params - The crawl progress parameters.
  * @param params.done - Number of URLs completed by the deal queue.
  * @param params.total - Total number of URLs in the deal queue (including completed).
  * @param params.resumeOffset - Offset from a previous resumed session.
  * @param params.externalTotal - Number of external URLs discovered.
  * @param params.externalDone - Number of external URLs completed.
+ * @param params.pagesScraped - Number of HTML pages rendered by the browser in this session.
  * @param params.limit - Number of parallel workers.
  * @returns The formatted progress string with ANSI color codes.
  */
-export declare function formatCrawlProgress({ done, total, resumeOffset, externalTotal, externalDone, limit, }: FormatCrawlProgressParams): string;
+export declare function formatCrawlProgress({ done, total, resumeOffset, externalTotal, externalDone, pagesScraped, limit, }: FormatCrawlProgressParams): string;
 export {};

package/lib/crawler/format-crawl-progress.js

@@ -1,19 +1,27 @@
 import c from 'ansi-colors';
+/**
+ * Number formatter for thousands-separated count display (e.g. `1,234,567`).
+ */
+const countFormat = new Intl.NumberFormat('en-US');
 /**
  * Formats the crawl progress header for the deal() progress display.
  *
- * Shows "done / found (remaining)" format instead of "done/total"
- * to make it clearer that the total is expected to grow during crawling.
+ * Shows "done(pages) / found URLs (remaining)" format instead of "done/total"
+ * to make it clearer that the total is expected to grow during crawling,
+ * that the counts are processed URLs (not resulting pages), and how many of
+ * those URLs were actually rendered by the browser as HTML pages.
+ * Counts are formatted with thousands separators (e.g. `1,234,567`).
  * @param params - The crawl progress parameters.
  * @param params.done - Number of URLs completed by the deal queue.
  * @param params.total - Total number of URLs in the deal queue (including completed).
  * @param params.resumeOffset - Offset from a previous resumed session.
  * @param params.externalTotal - Number of external URLs discovered.
  * @param params.externalDone - Number of external URLs completed.
+ * @param params.pagesScraped - Number of HTML pages rendered by the browser in this session.
  * @param params.limit - Number of parallel workers.
  * @returns The formatted progress string with ANSI color codes.
  */
-export function formatCrawlProgress({ done, total, resumeOffset, externalTotal, externalDone, limit, }) {
+export function formatCrawlProgress({ done, total, resumeOffset, externalTotal, externalDone, pagesScraped, limit, }) {
     const allDone = done + resumeOffset;
     const allTotal = total + resumeOffset;
     const internalDone = allDone - externalDone;
@@ -22,8 +30,8 @@ export function formatCrawlProgress({ done, total, resumeOffset, externalTotal,
     const externalRemaining = externalTotal - externalDone;
     const totalRemaining = internalRemaining + externalRemaining;
     const pct = allTotal > 0 ? Math.round((allDone / allTotal) * 100) : 0;
-    return (c.bold(`Crawling: ${internalDone} done / ${internalTotal} found`) +
-        c.dim(` (+${externalDone}/${externalTotal} ext)`) +
-        c.bold(` (${pct}%) [${totalRemaining} remaining]`) +
+    return (c.bold(`Crawling: ${countFormat.format(internalDone)}(${countFormat.format(pagesScraped)}) done / ${countFormat.format(internalTotal)} found URLs`) +
+        c.dim(` (+${countFormat.format(externalDone)}/${countFormat.format(externalTotal)} ext)`) +
+        c.bold(` (${pct}%) [${countFormat.format(totalRemaining)} remaining]`) +
         c.dim(` [${limit} parallel]`));
 }

package/lib/crawler/handle-browser-close.d.ts

@@ -0,0 +1,29 @@
+import type { ClosableBrowser } from './close-browser-safely.js';
+/**
+ * Debug-style logger compatible with the `debug` package's printf-style API.
+ *
+ * Declared structurally so the function stays unit-testable with a `vi.fn()`
+ * stub and free of a `debug` import.
+ */
+export type BrowserCloseLogger = (
+/** printf-style format string (e.g. `'%s'`, `'%O'`). */
+formatter: string, 
+/** Arguments interpolated into the format string. */
+...args: readonly unknown[]) => void;
+/**
+ * Closes a Puppeteer browser used to scrape a single URL, recording any
+ * timeout fallback or unexpected cleanup error to the supplied logger.
+ *
+ * WHY a dedicated function: the prior inline `finally` in
+ * {@link Crawler.#launchBrowserAndScrape} mixed cleanup orchestration with
+ * production log formatting. Splitting it out makes the two observable
+ * branches (force-kill notice and unexpected-error notice) directly testable
+ * without spawning a real browser, and keeps the rule that a finally block
+ * never throws: any error from {@link closeBrowserSafely} is logged here,
+ * never re-thrown.
+ * @param browser - The browser to close.
+ * @param urlHref - URL string included in the log messages for diagnostic
+ *   context.
+ * @param log - Logger used to record timeout and error events.
+ */
+export declare function handleBrowserClose(browser: ClosableBrowser, urlHref: string, log: BrowserCloseLogger): Promise<void>;

package/lib/crawler/handle-browser-close.js

@@ -0,0 +1,28 @@
+import { closeBrowserSafely } from './close-browser-safely.js';
+/**
+ * Closes a Puppeteer browser used to scrape a single URL, recording any
+ * timeout fallback or unexpected cleanup error to the supplied logger.
+ *
+ * WHY a dedicated function: the prior inline `finally` in
+ * {@link Crawler.#launchBrowserAndScrape} mixed cleanup orchestration with
+ * production log formatting. Splitting it out makes the two observable
+ * branches (force-kill notice and unexpected-error notice) directly testable
+ * without spawning a real browser, and keeps the rule that a finally block
+ * never throws: any error from {@link closeBrowserSafely} is logged here,
+ * never re-thrown.
+ * @param browser - The browser to close.
+ * @param urlHref - URL string included in the log messages for diagnostic
+ *   context.
+ * @param log - Logger used to record timeout and error events.
+ */
+export async function handleBrowserClose(browser, urlHref, log) {
+    try {
+        const timedOut = await closeBrowserSafely(browser);
+        if (timedOut) {
+            log('Force-killed wedged Chromium browser for %s (close() timed out)', urlHref);
+        }
+    }
+    catch (error) {
+        log('closeBrowserSafely failed for %s: %O', urlHref, error);
+    }
+}

package/lib/crawler/is-html-content-type.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Determine whether a Content-Type media type is HTML.
+ *
+ * MIME types are case-insensitive (RFC 2045), and values captured from
+ * Puppeteer responses preserve the server's original casing, so the
+ * comparison must normalize case — `text/HTML` is HTML. Surrounding
+ * whitespace (e.g. `text/html ` left over after parameter stripping)
+ * is also tolerated.
+ *
+ * This is the single source of truth for HTML detection — `Page.isPage()`
+ * and the link list delegate here so the classification never diverges
+ * between code paths.
+ * @param contentType - The media type portion of a Content-Type header
+ *   (parameters already stripped), or `null` when unknown.
+ * @returns `true` when the media type is `text/html` in any letter case.
+ */
+export declare function isHtmlContentType(contentType: string | null): boolean;

package/lib/crawler/is-html-content-type.js

@@ -0,0 +1,19 @@
+/**
+ * Determine whether a Content-Type media type is HTML.
+ *
+ * MIME types are case-insensitive (RFC 2045), and values captured from
+ * Puppeteer responses preserve the server's original casing, so the
+ * comparison must normalize case — `text/HTML` is HTML. Surrounding
+ * whitespace (e.g. `text/html ` left over after parameter stripping)
+ * is also tolerated.
+ *
+ * This is the single source of truth for HTML detection — `Page.isPage()`
+ * and the link list delegate here so the classification never diverges
+ * between code paths.
+ * @param contentType - The media type portion of a Content-Type header
+ *   (parameters already stripped), or `null` when unknown.
+ * @returns `true` when the media type is `text/html` in any letter case.
+ */
+export function isHtmlContentType(contentType) {
+    return contentType !== null && contentType.trim().toLowerCase() === 'text/html';
+}

package/lib/crawler/is-likely-html-url.d.ts

@@ -0,0 +1,22 @@
+import type { ExURL } from '@d-zero/shared/parse-url';
+/**
+ * Heuristically decide whether a discovered URL is likely to resolve to an
+ * HTML page, based solely on the URL itself.
+ *
+ * WHY URL-only: this runs at enqueue time — before any HEAD/GET — so the actual
+ * `Content-Type` is unknown. The crawler uses the result to prioritise the
+ * dealer queue (likely-HTML URLs are `unshift`ed to the front so page crawling
+ * advances ahead of asset/document fetches), so a heuristic is acceptable: a
+ * misclassification only changes fetch order, never correctness.
+ *
+ * Classification rules:
+ * - Non-HTTP URLs (`mailto:`, `tel:`, …) are never HTML pages.
+ * - Extensionless / directory-style URLs (`/`, `/about/`), and bare trailing-dot
+ *   URLs (`/index.`, whose `extname` is `"."`), are treated as HTML — these are
+ *   the overwhelmingly common shape for navigable pages.
+ * - URLs whose extension is in {@link HTML_EXTENSIONS} are HTML; every other
+ *   extension (`.jpg`, `.pdf`, `.css`, `.js`, …) is treated as non-HTML.
+ * @param url - The parsed URL to classify.
+ * @returns `true` when the URL is likely an HTML page.
+ */
+export declare function isLikelyHtmlUrl(url: ExURL): boolean;

package/lib/crawler/is-likely-html-url.js

@@ -0,0 +1,65 @@
+/**
+ * File extensions (lowercased, leading dot included) that typically denote a
+ * document served as `text/html`. Both static pages (`.html`) and the common
+ * server-side template / handler extensions (`.php`, `.aspx`, `.jsp`, `.ashx`,
+ * `.jsf` …) are listed because they usually serve an HTML page.
+ *
+ * A few entries (`.cgi`, `.do`, `.action`) are ambiguous — they sometimes
+ * return JSON or binary — but are kept here because a misclassification only
+ * changes fetch order, never correctness (see {@link isLikelyHtmlUrl}).
+ *
+ * Keys keep the leading dot so they can be compared directly against
+ * `ExURL.extname` (Node `Path.extname` output) without stripping it.
+ */
+const HTML_EXTENSIONS = new Set([
+    '.html',
+    '.htm',
+    '.xhtml',
+    '.shtml',
+    '.mhtml',
+    '.php',
+    '.php3',
+    '.php4',
+    '.php5',
+    '.phtml',
+    '.asp',
+    '.aspx',
+    '.ashx',
+    '.jsp',
+    '.jspx',
+    '.jsf',
+    '.cfm',
+    '.cgi',
+    '.do',
+    '.action',
+]);
+/**
+ * Heuristically decide whether a discovered URL is likely to resolve to an
+ * HTML page, based solely on the URL itself.
+ *
+ * WHY URL-only: this runs at enqueue time — before any HEAD/GET — so the actual
+ * `Content-Type` is unknown. The crawler uses the result to prioritise the
+ * dealer queue (likely-HTML URLs are `unshift`ed to the front so page crawling
+ * advances ahead of asset/document fetches), so a heuristic is acceptable: a
+ * misclassification only changes fetch order, never correctness.
+ *
+ * Classification rules:
+ * - Non-HTTP URLs (`mailto:`, `tel:`, …) are never HTML pages.
+ * - Extensionless / directory-style URLs (`/`, `/about/`), and bare trailing-dot
+ *   URLs (`/index.`, whose `extname` is `"."`), are treated as HTML — these are
+ *   the overwhelmingly common shape for navigable pages.
+ * - URLs whose extension is in {@link HTML_EXTENSIONS} are HTML; every other
+ *   extension (`.jpg`, `.pdf`, `.css`, `.js`, …) is treated as non-HTML.
+ * @param url - The parsed URL to classify.
+ * @returns `true` when the URL is likely an HTML page.
+ */
+export function isLikelyHtmlUrl(url) {
+    if (!url.isHTTP) {
+        return false;
+    }
+    const extname = url.extname;
+    if (!extname || extname === '.') {
+        return true;
+    }
+    return HTML_EXTENSIONS.has(extname.toLowerCase());
+}