@nitpicker/crawler 0.9.0 → 0.12.0

package/lib/crawler/plan-sub-resource-emits.d.ts

@@ -0,0 +1,63 @@
+import type { CrawlerEventTypes } from './types.js';
+import type { PageSource } from '../archive/types.js';
+import type { ResourceEntry } from '@d-zero/beholder';
+/**
+ * Planned `response` emit produced by {@link planSubResourceEmits}.
+ */
+export interface PlannedResponseEmit {
+    /** The resource payload to attach to the `response` event. */
+    resource: CrawlerEventTypes['response']['resource'];
+    /**
+     * The `source` field propagated to the `response` event. Resolved from
+     * the parent page's lineage via {@link deriveResourceSource} — pinning
+     * this through a planning step (rather than computing it inline in
+     * `#handleResources`) lets the wire-up be unit-tested without spinning
+     * up the puppeteer mock stack.
+     */
+    source: PageSource | undefined;
+}
+/**
+ * Planned `responseReferrers` emit produced by {@link planSubResourceEmits}.
+ * Always emitted, regardless of whether the resource is new — `isNew` only
+ * gates the `response` event.
+ */
+export interface PlannedReferrerEmit {
+    /** The page URL that triggered the sub-resource fetch. */
+    url: string;
+    /** The resource URL (hash stripped to match the storage key). */
+    src: string;
+}
+/**
+ * Output of {@link planSubResourceEmits}: the deduped `response` plan and
+ * the per-resource `responseReferrers` plan, side-by-side.
+ */
+export interface SubResourceEmitPlan {
+    /** `response` events to emit (new resources only). */
+    responseEmits: PlannedResponseEmit[];
+    /** `responseReferrers` events to emit (every resource, even seen ones). */
+    referrerEmits: PlannedReferrerEmit[];
+}
+/**
+ * Decide which sub-resource `response` / `responseReferrers` events the
+ * crawler should emit for a page render, with the parent's source lineage
+ * baked into every `response` event's `source` field.
+ *
+ * Pure function — takes the resources captured during the render plus the
+ * seen-resource set and the parent's source, returns the emit plan. The
+ * caller (`Crawler.#handleResources`) is responsible for iterating the
+ * plan through its event emitter. Splitting "decide what to emit" from
+ * "actually emit" is what makes the lineage propagation contract
+ * unit-testable: the previous shape inlined `emit('response', { ...
+ * source: deriveResourceSource(...) })` and could only be exercised via a
+ * full scrape with a mocked puppeteer stack, which left the `source`
+ * value half of the contract effectively un-pinned.
+ *
+ * Mutates `seenResources` as a side effect — every captured resource is
+ * recorded as seen so the next call dedupes correctly. This mirrors the
+ * `Crawler.#resources` Set semantics that the planner is designed to share.
+ * @param resources - Sub-resource entries captured during the page render.
+ * @param parentSource - Merged source of the page being rendered, as resolved by `Crawler.#resolveParentSource`.
+ * @param seenResources - Mutable set of already-seen resource keys (mutated in place).
+ * @returns The plan of `response` + `responseReferrers` emits to dispatch.
+ */
+export declare function planSubResourceEmits(resources: ResourceEntry[], parentSource: PageSource | undefined, seenResources: Set<string>): SubResourceEmitPlan;

package/lib/crawler/plan-sub-resource-emits.js

@@ -0,0 +1,44 @@
+import { deriveResourceSource } from './derive-resource-source.js';
+import { handleResourceResponse } from './handle-resource-response.js';
+/**
+ * Decide which sub-resource `response` / `responseReferrers` events the
+ * crawler should emit for a page render, with the parent's source lineage
+ * baked into every `response` event's `source` field.
+ *
+ * Pure function — takes the resources captured during the render plus the
+ * seen-resource set and the parent's source, returns the emit plan. The
+ * caller (`Crawler.#handleResources`) is responsible for iterating the
+ * plan through its event emitter. Splitting "decide what to emit" from
+ * "actually emit" is what makes the lineage propagation contract
+ * unit-testable: the previous shape inlined `emit('response', { ...
+ * source: deriveResourceSource(...) })` and could only be exercised via a
+ * full scrape with a mocked puppeteer stack, which left the `source`
+ * value half of the contract effectively un-pinned.
+ *
+ * Mutates `seenResources` as a side effect — every captured resource is
+ * recorded as seen so the next call dedupes correctly. This mirrors the
+ * `Crawler.#resources` Set semantics that the planner is designed to share.
+ * @param resources - Sub-resource entries captured during the page render.
+ * @param parentSource - Merged source of the page being rendered, as resolved by `Crawler.#resolveParentSource`.
+ * @param seenResources - Mutable set of already-seen resource keys (mutated in place).
+ * @returns The plan of `response` + `responseReferrers` emits to dispatch.
+ */
+export function planSubResourceEmits(resources, parentSource, seenResources) {
+    const subResourceSource = deriveResourceSource(parentSource);
+    const responseEmits = [];
+    const referrerEmits = [];
+    for (const { resource, pageUrl } of resources) {
+        const { isNew } = handleResourceResponse(resource, seenResources);
+        if (isNew) {
+            responseEmits.push({
+                resource: resource,
+                source: subResourceSource,
+            });
+        }
+        referrerEmits.push({
+            url: pageUrl,
+            src: resource.url.withoutHash,
+        });
+    }
+    return { responseEmits, referrerEmits };
+}

package/lib/crawler/preload-short-circuit-error.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Thrown by `Crawler.#sendHeadRequest` when the target URL's hostname is in
+ * `dnsBurnedHostCache` — both session-learned and preload-seeded burns
+ * land here.
+ *
+ * The orchestrator's `crawler.on('error', …)` handler tests `instanceof` and
+ * skips writing this error to `crawl_errors` / `error.log`, so the same
+ * preload data isn't re-amplified on subsequent crawls. `pages.status = -1`
+ * still gets set through the normal scrape-error path.
+ *
+ * The message embeds the `ENOTFOUND` token so any downstream consumer that
+ * runs `classifyErrorKind` over it (e.g. dealer log forwarders) still gets
+ * the `'dns'` classification.
+ */
+export declare class PreloadShortCircuitError extends Error {
+    /** Sniffable flag for callers that prefer duck-typing over instanceof. */
+    readonly isPreloadShortCircuit: true;
+    /**
+     * @param host - The DNS-burned hostname (already lowercased / Punycoded).
+     */
+    constructor(host: string);
+}

package/lib/crawler/preload-short-circuit-error.js

@@ -0,0 +1,25 @@
+/**
+ * Thrown by `Crawler.#sendHeadRequest` when the target URL's hostname is in
+ * `dnsBurnedHostCache` — both session-learned and preload-seeded burns
+ * land here.
+ *
+ * The orchestrator's `crawler.on('error', …)` handler tests `instanceof` and
+ * skips writing this error to `crawl_errors` / `error.log`, so the same
+ * preload data isn't re-amplified on subsequent crawls. `pages.status = -1`
+ * still gets set through the normal scrape-error path.
+ *
+ * The message embeds the `ENOTFOUND` token so any downstream consumer that
+ * runs `classifyErrorKind` over it (e.g. dealer log forwarders) still gets
+ * the `'dns'` classification.
+ */
+export class PreloadShortCircuitError extends Error {
+    /** Sniffable flag for callers that prefer duck-typing over instanceof. */
+    isPreloadShortCircuit = true;
+    /**
+     * @param host - The DNS-burned hostname (already lowercased / Punycoded).
+     */
+    constructor(host) {
+        super(`getaddrinfo ENOTFOUND ${host}`);
+        this.name = 'PreloadShortCircuitError';
+    }
+}

package/lib/crawler/redirect-dest-key.d.ts

@@ -0,0 +1,19 @@
+import type { ExURL } from '@d-zero/shared/parse-url';
+/**
+ * Computes the protocol-agnostic dedup key for the final destination a request
+ * lands on after following its redirect chain.
+ *
+ * Used by the redirect-convergence optimisation (#73): the crawler remembers
+ * which final destinations have already been rendered, keyed by this value, so
+ * that many source URLs all redirecting to one destination render it only once.
+ * When there is no redirect, the destination is the requested URL itself.
+ *
+ * The key matches the form used elsewhere in the crawler (`protocolAgnosticKey`
+ * over the normalised URL without hash/auth) so HTTP and HTTPS variants of the
+ * same destination collapse to one entry.
+ * @param url - The originally requested URL.
+ * @param redirectPaths - The redirect hop URLs captured during the HEAD
+ *   pre-flight, in order. The last entry is the final destination.
+ * @returns The protocol-agnostic key of the final destination.
+ */
+export declare function redirectDestKey(url: ExURL, redirectPaths: readonly string[]): string;

package/lib/crawler/redirect-dest-key.js

@@ -0,0 +1,27 @@
+import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
+import { protocolAgnosticKey } from './protocol-agnostic-key.js';
+/**
+ * Computes the protocol-agnostic dedup key for the final destination a request
+ * lands on after following its redirect chain.
+ *
+ * Used by the redirect-convergence optimisation (#73): the crawler remembers
+ * which final destinations have already been rendered, keyed by this value, so
+ * that many source URLs all redirecting to one destination render it only once.
+ * When there is no redirect, the destination is the requested URL itself.
+ *
+ * The key matches the form used elsewhere in the crawler (`protocolAgnosticKey`
+ * over the normalised URL without hash/auth) so HTTP and HTTPS variants of the
+ * same destination collapse to one entry.
+ * @param url - The originally requested URL.
+ * @param redirectPaths - The redirect hop URLs captured during the HEAD
+ *   pre-flight, in order. The last entry is the final destination.
+ * @returns The protocol-agnostic key of the final destination.
+ */
+export function redirectDestKey(url, redirectPaths) {
+    const last = redirectPaths.at(-1);
+    if (last === undefined) {
+        return protocolAgnosticKey(url.withoutHashAndAuth);
+    }
+    const parsed = parseUrl(last);
+    return protocolAgnosticKey(parsed ? parsed.withoutHashAndAuth : last);
+}

package/lib/crawler/resource-to-page-data.d.ts

@@ -0,0 +1,28 @@
+import type { ResourceLookupResult } from './types.js';
+import type { PageData } from '../utils/types/types.js';
+import type { ExURL } from '@d-zero/shared/parse-url';
+/**
+ * Parameters for {@link resourceToPageData}.
+ */
+interface ResourceToPageDataParams {
+    /** The queued URL being processed. */
+    readonly url: ExURL;
+    /** Whether the URL is external to the crawl scope. */
+    readonly isExternal: boolean;
+    /** The recorded sub-resource data captured during page rendering. */
+    readonly resource: ResourceLookupResult;
+}
+/**
+ * Synthesize {@link PageData} from a recorded sub-resource row, or return
+ * `null` when the resource is not eligible for reuse.
+ *
+ * Eligible: the status is 2xx AND the content type is known and not
+ * `text/html`. Non-2xx rows (redirect hops, errors, 304s) and HTML rows must
+ * fall back to the normal HEAD pre-flight / browser scrape, which also
+ * guarantees that `redirectPaths: []` here is accurate — a URL that
+ * redirects is recorded with its 3xx status and never reaches this path.
+ * @param params - The queued URL, its external flag, and the recorded resource data.
+ * @returns The synthesized page data, or `null` when the caller must fall back.
+ */
+export declare function resourceToPageData(params: ResourceToPageDataParams): PageData | null;
+export {};

package/lib/crawler/resource-to-page-data.js

@@ -0,0 +1,59 @@
+import { isHtmlContentType } from './is-html-content-type.js';
+/**
+ * Synthesize {@link PageData} from a recorded sub-resource row, or return
+ * `null` when the resource is not eligible for reuse.
+ *
+ * Eligible: the status is 2xx AND the content type is known and not
+ * `text/html`. Non-2xx rows (redirect hops, errors, 304s) and HTML rows must
+ * fall back to the normal HEAD pre-flight / browser scrape, which also
+ * guarantees that `redirectPaths: []` here is accurate — a URL that
+ * redirects is recorded with its 3xx status and never reaches this path.
+ * @param params - The queued URL, its external flag, and the recorded resource data.
+ * @returns The synthesized page data, or `null` when the caller must fall back.
+ */
+export function resourceToPageData(params) {
+    const { url, isExternal, resource } = params;
+    if (resource.status == null ||
+        resource.status < 200 ||
+        resource.status >= 300 ||
+        resource.contentType == null ||
+        isHtmlContentType(resource.contentType)) {
+        return null;
+    }
+    return {
+        url,
+        redirectPaths: [],
+        isTarget: !isExternal,
+        isExternal,
+        status: resource.status,
+        statusText: resource.statusText ?? '',
+        contentType: resource.contentType,
+        contentLength: resource.contentLength,
+        responseHeaders: resource.responseHeaders,
+        // beholder 3.0.0 made jsonLd / speculationRules / tags / others /
+        // originTrial required Meta fields. This fallback wraps a non-HTML
+        // resource (PDF, image, etc.) that was discovered via subresource
+        // inspection rather than full scraping; populate every required slot
+        // so downstream insert/derive helpers iterate without crashing.
+        meta: {
+            title: '',
+            jsonLd: [],
+            speculationRules: [],
+            tags: { detected: {}, entries: [] },
+            others: {
+                meta: {},
+                property: {},
+                httpEquiv: {},
+                itemprop: {},
+                link: [],
+                script: [],
+                iframe: [],
+            },
+            originTrial: [],
+        },
+        anchorList: [],
+        imageList: [],
+        html: '',
+        isSkipped: false,
+    };
+}

package/lib/crawler/should-burn-host.d.ts

@@ -0,0 +1,78 @@
+import type { ErrorKind } from '../types.js';
+/**
+ * Inputs to {@link shouldBurnHost}.
+ */
+export interface ShouldBurnHostParams {
+    /**
+     * The {@link ErrorKind} classified from the final-attempt error message
+     * (i.e. the error that ended the retry loop in `Crawler.#sendHeadRequest`'s
+     * `onGiveUp`).
+     */
+    errorKind: ErrorKind;
+    /**
+     * Lower-cased hostname whose URL just exhausted retries. Must already be
+     * normalised by the caller — `dnsBurnedHostCache` keys are
+     * `url.hostname.toLowerCase()`, so this guard reuses that exact form to
+     * stay consistent across the two sites.
+     */
+    host: string;
+    /**
+     * Set of hostnames that have had at least one successful
+     * `fetchDestination` response in this session. `ReadonlySet` because the
+     * decision is read-only — populating the set is the caller's job.
+     */
+    successfulHosts: ReadonlySet<string>;
+}
+/**
+ * Burn the host iff the final-attempt kind is `'dns'` AND the host has no
+ * session-success record. Pure function — unit-testable in isolation from
+ * the `Crawler` instance, the in-memory caches, and the dealer's retry
+ * plumbing.
+ *
+ * **Why the session-success gate exists**: the first worker to exhaust
+ * retries with `getaddrinfo ENOTFOUND` would otherwise burn the host and
+ * make every subsequent URL on it short-circuit immediately via
+ * `PreloadShortCircuitError`, draining the dealer's work queue in seconds
+ * and collapsing the crawl into a degenerate `crawlEnd`. When the cause is
+ * a local-network blip (operator's WiFi → tethering / VPN flip / ISP DNS
+ * hiccup mid-crawl) rather than a dead domain, the host was demonstrably
+ * alive moments earlier — earlier successes on it are recorded in
+ * `successfulHosts`, so the cascade is suppressed.
+ *
+ * **Why `'dns-transient'` is excluded**: `EAI_AGAIN` / `EREFUSED` are
+ * absorbed by the retry layer within the session; a final-attempt
+ * `'dns-transient'` is rare enough that we'd rather pay the per-URL retry
+ * cost than wrongly fast-fail a healthy host that flapped briefly. Only the
+ * stronger `'dns'` kind is a candidate for burning.
+ *
+ * **Known limitation — first-URL false positives**: a host whose very first
+ * URL of the session hits a real network blip exhausts its retry budget
+ * before any URL has succeeded, so `successfulHosts` is still empty and the
+ * host IS burned. That URL's siblings on the same host then short-circuit.
+ * Acceptable trade-off: the alternative ("never burn anything") regresses
+ * the dead-domain fast-fail behavior that the burn cache exists for. The
+ * pause-dealer-on-outage layer (separate issue) covers this gap by
+ * detecting the outage BEFORE the first retry budget runs out.
+ *
+ * **Known limitation — preload-seeded burns are not un-burned**: a host
+ * added by `#preloadDnsBurnedHostCache` from the archive's `crawl_errors`
+ * trips `PreloadShortCircuitError` at the top of `#sendHeadRequest` and
+ * never reaches `fetchDestination`, so `successfulHosts` is never
+ * populated for it in this session. This is intentional — preload only
+ * seeds hosts whose archive evidence is "DNS-failed with no recovery", so
+ * un-burning them on a single transient success could re-introduce the
+ * cascade we are trying to prevent.
+ *
+ * **What does NOT contribute to `successfulHosts`**: external pages
+ * traversed via `fetchExternal: false` skip the HEAD pre-flight entirely
+ * (the crawler stamps a synthetic `PageData` without touching the
+ * network), so the host is not recorded as alive even if the same host
+ * appears in the crawled-internal scope. Callers must populate the set
+ * from real HTTP-response observations only.
+ * @param params - See {@link ShouldBurnHostParams}.
+ * @param params.errorKind
+ * @param params.host
+ * @param params.successfulHosts
+ * @returns `true` if the burn cache should record this host, `false` otherwise.
+ */
+export declare function shouldBurnHost({ errorKind, host, successfulHosts, }: ShouldBurnHostParams): boolean;

package/lib/crawler/should-burn-host.js

@@ -0,0 +1,61 @@
+/**
+ * Burn the host iff the final-attempt kind is `'dns'` AND the host has no
+ * session-success record. Pure function — unit-testable in isolation from
+ * the `Crawler` instance, the in-memory caches, and the dealer's retry
+ * plumbing.
+ *
+ * **Why the session-success gate exists**: the first worker to exhaust
+ * retries with `getaddrinfo ENOTFOUND` would otherwise burn the host and
+ * make every subsequent URL on it short-circuit immediately via
+ * `PreloadShortCircuitError`, draining the dealer's work queue in seconds
+ * and collapsing the crawl into a degenerate `crawlEnd`. When the cause is
+ * a local-network blip (operator's WiFi → tethering / VPN flip / ISP DNS
+ * hiccup mid-crawl) rather than a dead domain, the host was demonstrably
+ * alive moments earlier — earlier successes on it are recorded in
+ * `successfulHosts`, so the cascade is suppressed.
+ *
+ * **Why `'dns-transient'` is excluded**: `EAI_AGAIN` / `EREFUSED` are
+ * absorbed by the retry layer within the session; a final-attempt
+ * `'dns-transient'` is rare enough that we'd rather pay the per-URL retry
+ * cost than wrongly fast-fail a healthy host that flapped briefly. Only the
+ * stronger `'dns'` kind is a candidate for burning.
+ *
+ * **Known limitation — first-URL false positives**: a host whose very first
+ * URL of the session hits a real network blip exhausts its retry budget
+ * before any URL has succeeded, so `successfulHosts` is still empty and the
+ * host IS burned. That URL's siblings on the same host then short-circuit.
+ * Acceptable trade-off: the alternative ("never burn anything") regresses
+ * the dead-domain fast-fail behavior that the burn cache exists for. The
+ * pause-dealer-on-outage layer (separate issue) covers this gap by
+ * detecting the outage BEFORE the first retry budget runs out.
+ *
+ * **Known limitation — preload-seeded burns are not un-burned**: a host
+ * added by `#preloadDnsBurnedHostCache` from the archive's `crawl_errors`
+ * trips `PreloadShortCircuitError` at the top of `#sendHeadRequest` and
+ * never reaches `fetchDestination`, so `successfulHosts` is never
+ * populated for it in this session. This is intentional — preload only
+ * seeds hosts whose archive evidence is "DNS-failed with no recovery", so
+ * un-burning them on a single transient success could re-introduce the
+ * cascade we are trying to prevent.
+ *
+ * **What does NOT contribute to `successfulHosts`**: external pages
+ * traversed via `fetchExternal: false` skip the HEAD pre-flight entirely
+ * (the crawler stamps a synthetic `PageData` without touching the
+ * network), so the host is not recorded as alive even if the same host
+ * appears in the crawled-internal scope. Callers must populate the set
+ * from real HTTP-response observations only.
+ * @param params - See {@link ShouldBurnHostParams}.
+ * @param params.errorKind
+ * @param params.host
+ * @param params.successfulHosts
+ * @returns `true` if the burn cache should record this host, `false` otherwise.
+ */
+export function shouldBurnHost({ errorKind, host, successfulHosts, }) {
+    if (errorKind !== 'dns') {
+        return false;
+    }
+    if (successfulHosts.has(host)) {
+        return false;
+    }
+    return true;
+}

package/lib/crawler/should-get-fallback-on-head-failure.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Decide whether a HEAD pre-flight failure should trigger a GET retry and
+ * stay OUT of `destinationCache`.
+ *
+ * The two contracts are intentionally tied to one helper: anything we'll GET
+ * later because the HEAD answer might be wrong is also the same thing we
+ * must NOT freeze into the per-session cache (or the second attempt would
+ * hit the stale cached failure and skip the live retry that
+ * `Crawler.#sendHeadRequest`'s `HEAD_TIMEOUT_ESCALATION_MS` is supposed to
+ * pay for).
+ *
+ * The eligible kinds are:
+ *
+ * - **`NetTimeoutError`** — the HEAD pre-flight race fired without a
+ *   server response. The escalating retry can still succeed against a
+ *   slow-but-reachable host.
+ * - **`parse-error`** — `Parse Error` / `Expected HTTP/` / `Unexpected end
+ *   of stream`. Usually a middlebox rewriting / truncating the HEAD reply
+ *   the GET path traverses differently.
+ * - **`connection-reset`** — `ECONNRESET` / `ERR_CONNECTION_RESET` etc.
+ *   middlebox dropping the connection mid-response; a retry frequently
+ *   succeeds.
+ *
+ * Everything else — DNS, TLS, refused, blocked, plain timeout — is treated
+ * as a persistent within-session verdict and IS cached so repeated calls on
+ * the same host pay the network cost once.
+ * @param error - The `Error` raised by the HEAD attempt.
+ * @returns `true` when the error warrants a GET fallback AND a cache skip.
+ * @example
+ * ```ts
+ * shouldGetFallbackOnHeadFailure(new NetTimeoutError(url)); // true
+ * shouldGetFallbackOnHeadFailure(new Error('read ECONNRESET')); // true
+ * shouldGetFallbackOnHeadFailure(new Error('Parse Error')); // true
+ * shouldGetFallbackOnHeadFailure(new Error('getaddrinfo ENOTFOUND host')); // false
+ * shouldGetFallbackOnHeadFailure(new Error('ERR_CERT_DATE_INVALID')); // false
+ * ```
+ */
+export declare function shouldGetFallbackOnHeadFailure(error: Error): boolean;

package/lib/crawler/should-get-fallback-on-head-failure.js

@@ -0,0 +1,46 @@
+import { classifyErrorKind } from '../classify-error-kind.js';
+import NetTimeoutError from './net-timeout-error.js';
+/**
+ * Decide whether a HEAD pre-flight failure should trigger a GET retry and
+ * stay OUT of `destinationCache`.
+ *
+ * The two contracts are intentionally tied to one helper: anything we'll GET
+ * later because the HEAD answer might be wrong is also the same thing we
+ * must NOT freeze into the per-session cache (or the second attempt would
+ * hit the stale cached failure and skip the live retry that
+ * `Crawler.#sendHeadRequest`'s `HEAD_TIMEOUT_ESCALATION_MS` is supposed to
+ * pay for).
+ *
+ * The eligible kinds are:
+ *
+ * - **`NetTimeoutError`** — the HEAD pre-flight race fired without a
+ *   server response. The escalating retry can still succeed against a
+ *   slow-but-reachable host.
+ * - **`parse-error`** — `Parse Error` / `Expected HTTP/` / `Unexpected end
+ *   of stream`. Usually a middlebox rewriting / truncating the HEAD reply
+ *   the GET path traverses differently.
+ * - **`connection-reset`** — `ECONNRESET` / `ERR_CONNECTION_RESET` etc.
+ *   middlebox dropping the connection mid-response; a retry frequently
+ *   succeeds.
+ *
+ * Everything else — DNS, TLS, refused, blocked, plain timeout — is treated
+ * as a persistent within-session verdict and IS cached so repeated calls on
+ * the same host pay the network cost once.
+ * @param error - The `Error` raised by the HEAD attempt.
+ * @returns `true` when the error warrants a GET fallback AND a cache skip.
+ * @example
+ * ```ts
+ * shouldGetFallbackOnHeadFailure(new NetTimeoutError(url)); // true
+ * shouldGetFallbackOnHeadFailure(new Error('read ECONNRESET')); // true
+ * shouldGetFallbackOnHeadFailure(new Error('Parse Error')); // true
+ * shouldGetFallbackOnHeadFailure(new Error('getaddrinfo ENOTFOUND host')); // false
+ * shouldGetFallbackOnHeadFailure(new Error('ERR_CERT_DATE_INVALID')); // false
+ * ```
+ */
+export function shouldGetFallbackOnHeadFailure(error) {
+    if (error instanceof NetTimeoutError) {
+        return true;
+    }
+    const kind = classifyErrorKind(error.message);
+    return kind === 'parse-error' || kind === 'connection-reset';
+}