@nitpicker/crawler 0.11.0 → 0.12.0

package/lib/classify-error-kind.js

@@ -0,0 +1,122 @@
+/**
+ * Ordered message matchers. The first pattern that matches wins, so more
+ * specific transport causes (DNS, TLS, connection-*) are tested before the
+ * broader `protocol` / `timeout` buckets — e.g. `ETIMEDOUT` must classify as
+ * `connection-timeout`, not the page-level `timeout`, and a puppeteer
+ * `Protocol error` must not be swallowed by the `timeout` matcher.
+ */
+const MATCHERS = [
+    // `dns-transient` must be evaluated before `dns`: an `EAI_AGAIN` line also
+    // carries the `getaddrinfo` token, so the more specific transient pattern
+    // has to win. Splitting it out from `dns` keeps the DNS-burned host cache
+    // (which marks on `kind === 'dns'`) from punishing a host whose only sin
+    // was a local resolver hiccup.
+    { kind: 'dns-transient', pattern: /EAI_AGAIN|\bEREFUSED\b/i },
+    {
+        kind: 'dns',
+        pattern: /ENOTFOUND|getaddrinfo|ERR_NAME_NOT_RESOLVED|ERR_NAME_RESOLUTION_FAILED/i,
+    },
+    {
+        kind: 'tls',
+        // `Hostname/IP does not match certificate's altnames` is Node's
+        // node:tls hostname mismatch error and is emphatically a TLS issue;
+        // adding it here (alongside the OpenSSL / Chromium tokens) keeps
+        // hosts that serve the wrong-name cert (common with misconfigured
+        // edge / load-balancer setups) out of `unknown`. `altnames` is
+        // anchored to the preceding `certificate` token so a request whose
+        // error message merely mentions a path containing the substring
+        // `altnames` (e.g. `https://api.example.com/altnames/lookup` in a
+        // 5xx body) does NOT get misclassified into `tls` (which is a
+        // `PERMANENT_ERROR_KINDS` member — a false-positive would
+        // permanently exclude that page from `--retry-failed`).
+        pattern: /ERR_CERT|ERR_SSL|\bCERT_|SSL routines|ERR_BAD_SSL|UNABLE_TO_VERIFY|unable to verify|self.signed certificate|certificate has expired\s*$|\bERR_TLS|Hostname\/IP does not match certificate|certificate'?s? altnames/i,
+    },
+    { kind: 'connection-refused', pattern: /ECONNREFUSED|ERR_CONNECTION_REFUSED/i },
+    {
+        kind: 'connection-reset',
+        pattern: /ECONNRESET|socket hang up|ERR_CONNECTION_RESET|ERR_CONNECTION_CLOSED|ERR_EMPTY_RESPONSE/i,
+    },
+    {
+        kind: 'connection-timeout',
+        pattern: /ETIMEDOUT|ERR_CONNECTION_TIMED_OUT|ERR_TIMED_OUT/i,
+    },
+    // `local-network` is evaluated AFTER the connection-* matchers so a
+    // concrete cause (refused / reset / timeout) wins when both apply. Only
+    // "local network is unreachable / changed" symptoms — and the OS-level
+    // errors that surface them — land here. Short tokens (`EPIPE`, `EREFUSED`)
+    // are word-bounded so unrelated identifiers don't false-positive.
+    {
+        kind: 'local-network',
+        pattern: /ERR_INTERNET_DISCONNECTED|ERR_NETWORK_CHANGED|ERR_NETWORK_IO_SUSPENDED|ERR_ADDRESS_UNREACHABLE|ERR_NETWORK_UNREACHABLE|ENETUNREACH|EHOSTUNREACH|EADDRNOTAVAIL|ENOTCONN|\bEPIPE\b/i,
+    },
+    {
+        kind: 'parse-error',
+        pattern: /Parse Error|Expected HTTP\/|Unexpected end of stream/i,
+    },
+    // `client-blocked` covers Chromium's ERR_BLOCKED_* family — the browser
+    // actively decided to reject the request (ad/tracker heuristics, CSP,
+    // CORB / ORB, administrator block list, fingerprinting protection,
+    // cleartext policy, …). Per the upstream `net/base/net_error_list.h`,
+    // `ERR_BLOCKED_BY_CLIENT` is documented as "The client chose to block
+    // the request." — i.e. the server was never the deciding party. Listed
+    // before `protocol` so puppeteer's generic "Protocol error" wrapper
+    // (which sometimes embeds the underlying net error code) is correctly
+    // attributed to the blocked layer rather than the protocol layer.
+    {
+        kind: 'client-blocked',
+        pattern: /ERR_BLOCKED_BY_CLIENT|ERR_BLOCKED_BY_ADMINISTRATOR|ERR_BLOCKED_IN_INCOGNITO_BY_ADMINISTRATOR|ERR_BLOCKED_BY_RESPONSE|ERR_BLOCKED_BY_CSP|ERR_BLOCKED_BY_ORB|ERR_BLOCKED_BY_FINGERPRINTING_PROTECTION|ERR_CLEARTEXT_NOT_PERMITTED|ERR_NETWORK_ACCESS_REVOKED/i,
+    },
+    {
+        kind: 'protocol',
+        // `detached frame` is anchored to puppeteer's exact prefix
+        // `Attempted to use detached Frame` (its current Frame.ts
+        // emitter; the `i` flag below catches the lowercase variant
+        // automatically), not the bare two-token substring. The bare
+        // form would match unrelated diagnostics like a console message
+        // "detached frame ref leaked" echoed through a logger. The older
+        // Page-domain `frame (?:was |got )?detached` form is kept as a
+        // separate alternative because Chromium still surfaces that
+        // phrasing in some legacy code paths. Without one of these, the
+        // "Attempted to use detached Frame ..." messages observed on a
+        // real archive would slip into `unknown`.
+        pattern: /Protocol error|Target closed|Session closed|Execution context was destroyed|frame (?:was |got )?detached|Attempted to use detached frame|Navigating frame was detached|Cannot find context|Node with given id|Page\.\w+ returned/i,
+    },
+    {
+        kind: 'timeout',
+        // `Timeout: https?:` matches the NetTimeoutError "Timeout: <url>"
+        // form. Looking for the URL-shaped tail (rather than anchoring at
+        // line start) is what lets us catch the beholder-wrapped variant
+        // `[Retried N times] Timeout: https://...` that gets stored in
+        // `crawl_errors` / `error.log` after retry exhaustion — the bare
+        // `^Timeout:` form would only fire on the immediate failure and
+        // miss every retry-exhausted record (the ones that actually land
+        // in the archive). Required for slow-server timeouts that
+        // previously fell into `unknown`.
+        pattern: /Race \d|Navigation timeout|timeout of \d+\s*ms exceeded|TimeoutError|Timed out|Timeout: https?:/i,
+    },
+];
+/**
+ * Classify a raw crawler/scraper error message into a coarse {@link ErrorKind}.
+ *
+ * Pure and deterministic: the same message always yields the same kind, which
+ * is why the kind is derived on read rather than persisted — it can be applied
+ * uniformly to freshly captured `crawl_errors`, legacy `error.log` lines, and
+ * `page_errors` alike.
+ * @param message - The raw error message (a single line is sufficient; the
+ *   cause token such as `ENOTFOUND` or `Navigation timeout` lives there).
+ * @returns The matched kind, or `unknown` when no matcher applies.
+ * @example
+ * ```ts
+ * classifyErrorKind('getaddrinfo ENOTFOUND www.example.com'); // 'dns'
+ * classifyErrorKind('gave up after 3 retries — Race 180,000ms'); // 'timeout'
+ * classifyErrorKind('Protocol error (Page.reload): Target closed'); // 'protocol'
+ * ```
+ */
+export function classifyErrorKind(message) {
+    for (const { kind, pattern } of MATCHERS) {
+        if (pattern.test(message)) {
+            return kind;
+        }
+    }
+    return 'unknown';
+}

package/lib/crawler/build-js-redirect-edge.d.ts

@@ -0,0 +1,68 @@
+import type { RedirectEdgeResult } from './types.js';
+import type { PageData } from '@d-zero/beholder';
+import type { ExURL } from '@d-zero/shared/parse-url';
+/**
+ * Parameters for {@link buildJsRedirectEdge}.
+ */
+export interface BuildJsRedirectEdgeParams {
+    /** Source URL puppeteer was asked to navigate to. */
+    url: ExURL;
+    /**
+     * Whether `url` is external to the crawl scope. Propagated onto the
+     * synthesised `PageData` so downstream `linkList.done` classifies the
+     * source the same way the original scrape would have.
+     */
+    isExternal: boolean;
+    /**
+     * Raw error message from `BrowserScrapeResult.error`. The rescue only
+     * fires when this matches the narrow `Page.goto returned null` sentinel
+     * (see {@link isJsRedirectErrorShape}) — anything else falls through
+     * so genuine browser failures surface unchanged.
+     */
+    errorMessage: string | null | undefined;
+    /**
+     * Value of `BrowserScrapeResult.postNavigationUrl` — the URL puppeteer's
+     * `page.url()` reported after the throw. Passed to
+     * {@link deriveJsRedirectTarget} for canonicalisation, credential strip,
+     * and scheme filtering.
+     */
+    postNavigationUrl: string | null | undefined;
+    /**
+     * When present, the rescue builds the redirect-edge `PageData` by
+     * spreading this HEAD-pre-flight result and overriding `redirectPaths`.
+     * Use this on the HEAD-success-then-puppeteer-fail path so the source row
+     * carries the real HTTP-level status / content-type from HEAD. When
+     * omitted, the rescue builds a synthetic placeholder via
+     * {@link linkToPageData} with `status = -1` / `statusText = errorMessage`
+     * — used on the HEAD-fail-then-puppeteer-fallback path where there is
+     * no HEAD response to draw from.
+     */
+    headCheckResult?: PageData;
+}
+/**
+ * Build a JS-redirect `RedirectEdgeResult` when the browser-scrape error
+ * looks like puppeteer's `page.goto() === null` shape AND `page.url()`
+ * resolved to a meaningfully-different URL.
+ *
+ * **Why this helper exists** — both rescue call sites in
+ * `Crawler.#scrapePage` (the HEAD-success-then-puppeteer-fail tail and the
+ * HEAD-fail-then-puppeteer-fallback branch inside `#sendHeadRequest`) run
+ * the same three-step recipe: classify the error shape → derive a JS target
+ * from `page.url()` → synthesise a redirect-edge result. Inlining the recipe
+ * twice means a future change to `deriveJsRedirectTarget`'s contract or the
+ * sentinel string has to be applied to both copies in sync; collapsing into
+ * a single helper keeps the rescue's invariants in one place. The
+ * intentional difference between the two call sites (whether to fold a
+ * HEAD result into the synthesised PageData or to start from a
+ * `linkToPageData` placeholder) is reduced to a single optional parameter
+ * (`headCheckResult`), preserving both shapes without behavioural drift.
+ *
+ * The helper logs at `crawlerLog` whenever it fires so operators tailing
+ * `DEBUG=Nitpicker:Crawler` see exactly which URL got rescued and to which
+ * destination — without this trail the rescue is invisible to anyone
+ * debugging "why did this 200-OK source row land in the archive as a 301?".
+ * @param params - Inputs gathered at the call site.
+ * @returns A `RedirectEdgeResult` when the rescue applies, otherwise
+ *   `null` (the caller should fall through to the existing error path).
+ */
+export declare function buildJsRedirectEdge(params: BuildJsRedirectEdgeParams): RedirectEdgeResult | null;

package/lib/crawler/build-js-redirect-edge.js

@@ -0,0 +1,57 @@
+import { crawlerLog } from '../debug.js';
+import { deriveJsRedirectTarget } from './derive-js-redirect-target.js';
+import { isJsRedirectErrorShape } from './is-js-redirect-error-shape.js';
+import { linkToPageData } from './link-to-page-data.js';
+/**
+ * Build a JS-redirect `RedirectEdgeResult` when the browser-scrape error
+ * looks like puppeteer's `page.goto() === null` shape AND `page.url()`
+ * resolved to a meaningfully-different URL.
+ *
+ * **Why this helper exists** — both rescue call sites in
+ * `Crawler.#scrapePage` (the HEAD-success-then-puppeteer-fail tail and the
+ * HEAD-fail-then-puppeteer-fallback branch inside `#sendHeadRequest`) run
+ * the same three-step recipe: classify the error shape → derive a JS target
+ * from `page.url()` → synthesise a redirect-edge result. Inlining the recipe
+ * twice means a future change to `deriveJsRedirectTarget`'s contract or the
+ * sentinel string has to be applied to both copies in sync; collapsing into
+ * a single helper keeps the rescue's invariants in one place. The
+ * intentional difference between the two call sites (whether to fold a
+ * HEAD result into the synthesised PageData or to start from a
+ * `linkToPageData` placeholder) is reduced to a single optional parameter
+ * (`headCheckResult`), preserving both shapes without behavioural drift.
+ *
+ * The helper logs at `crawlerLog` whenever it fires so operators tailing
+ * `DEBUG=Nitpicker:Crawler` see exactly which URL got rescued and to which
+ * destination — without this trail the rescue is invisible to anyone
+ * debugging "why did this 200-OK source row land in the archive as a 301?".
+ * @param params - Inputs gathered at the call site.
+ * @returns A `RedirectEdgeResult` when the rescue applies, otherwise
+ *   `null` (the caller should fall through to the existing error path).
+ */
+export function buildJsRedirectEdge(params) {
+    if (!isJsRedirectErrorShape(params.errorMessage)) {
+        return null;
+    }
+    const jsRedirectTarget = deriveJsRedirectTarget(params.url.withoutHashAndAuth, params.postNavigationUrl);
+    if (jsRedirectTarget === null) {
+        return null;
+    }
+    const pageData = params.headCheckResult
+        ? { ...params.headCheckResult, redirectPaths: [jsRedirectTarget] }
+        : linkToPageData({
+            url: params.url,
+            isExternal: params.isExternal,
+            isLowerLayer: false,
+            dest: {
+                redirectPaths: [jsRedirectTarget],
+                status: -1,
+                statusText: typeof params.errorMessage === 'string' ? params.errorMessage : '',
+                contentType: null,
+                contentLength: null,
+                responseHeaders: null,
+                title: '',
+            },
+        });
+    crawlerLog('JS-redirect rescue fired for %s → %s (HEAD %s)', params.url.href, jsRedirectTarget, params.headCheckResult ? 'available' : 'absent');
+    return { type: 'redirect-edge', source: 'js-redirect', pageData };
+}

package/lib/crawler/build-redirect-event.d.ts

@@ -0,0 +1,24 @@
+import type { CrawlerEventTypes, InventoryMode } from './types.js';
+import type { PageData } from '../utils/types/types.js';
+/**
+ * Build the payload for the {@link CrawlerEventTypes.redirect} event so the
+ * `{ result, source }` shape is constructed in one place. Pure function —
+ * no I/O, no side effects, just the wiring that connects
+ * {@link derivePageSource} to the emit call.
+ *
+ * Exists so a unit test can pin the wiring directly. Before extraction
+ * the assembly lived inline in `Crawler.#scrape`, which meant the only
+ * way to catch "someone deletes the `source` field" or "someone passes
+ * `inventoryMode` where it should pass `derivePageSource(...)`" was the
+ * inventory E2E. Now a single-purpose spec exercises the construction.
+ *
+ * The originating URL passed in `pageUrlWithoutHashAndAuth` must be the
+ * URL the redirect chain STARTS from (the page being scraped), not the
+ * destination — see {@link Database.recordRedirect} JSDoc for why
+ * lineage propagates from origin, not from the destination.
+ * @param pageData - HEAD-resolved page data carrying the redirect chain.
+ * @param inventoryMode - Inventory-mode config from `CrawlerOptions.inventoryMode`, or `null` outside `--inventory`.
+ * @param pageUrlWithoutHashAndAuth - The originating page URL keyed by `withoutHashAndAuth`.
+ * @returns The exact event payload to feed `emit('redirect', ...)`.
+ */
+export declare function buildRedirectEvent(pageData: PageData, inventoryMode: InventoryMode | null, pageUrlWithoutHashAndAuth: string): CrawlerEventTypes['redirect'];

package/lib/crawler/build-redirect-event.js

@@ -0,0 +1,28 @@
+import { derivePageSource } from './derive-page-source.js';
+/**
+ * Build the payload for the {@link CrawlerEventTypes.redirect} event so the
+ * `{ result, source }` shape is constructed in one place. Pure function —
+ * no I/O, no side effects, just the wiring that connects
+ * {@link derivePageSource} to the emit call.
+ *
+ * Exists so a unit test can pin the wiring directly. Before extraction
+ * the assembly lived inline in `Crawler.#scrape`, which meant the only
+ * way to catch "someone deletes the `source` field" or "someone passes
+ * `inventoryMode` where it should pass `derivePageSource(...)`" was the
+ * inventory E2E. Now a single-purpose spec exercises the construction.
+ *
+ * The originating URL passed in `pageUrlWithoutHashAndAuth` must be the
+ * URL the redirect chain STARTS from (the page being scraped), not the
+ * destination — see {@link Database.recordRedirect} JSDoc for why
+ * lineage propagates from origin, not from the destination.
+ * @param pageData - HEAD-resolved page data carrying the redirect chain.
+ * @param inventoryMode - Inventory-mode config from `CrawlerOptions.inventoryMode`, or `null` outside `--inventory`.
+ * @param pageUrlWithoutHashAndAuth - The originating page URL keyed by `withoutHashAndAuth`.
+ * @returns The exact event payload to feed `emit('redirect', ...)`.
+ */
+export function buildRedirectEvent(pageData, inventoryMode, pageUrlWithoutHashAndAuth) {
+    return {
+        result: pageData,
+        source: derivePageSource(inventoryMode, pageUrlWithoutHashAndAuth),
+    };
+}

package/lib/crawler/clear-dns-burned-host-cache.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Clears the DNS-burned host cache and resets the short-circuit counter.
+ * Called between crawl sessions, alongside {@link clearDestinationCache}, to
+ * avoid leaking state from a previous session into a fresh one.
+ */
+export declare function clearDnsBurnedHostCache(): void;

package/lib/crawler/clear-dns-burned-host-cache.js

@@ -0,0 +1,11 @@
+import { dnsBurnedHostCache } from './dns-burned-host-cache.js';
+import { dnsBurnedHostShortCircuitCounter } from './dns-burned-host-short-circuit-counter.js';
+/**
+ * Clears the DNS-burned host cache and resets the short-circuit counter.
+ * Called between crawl sessions, alongside {@link clearDestinationCache}, to
+ * avoid leaking state from a previous session into a fresh one.
+ */
+export function clearDnsBurnedHostCache() {
+    dnsBurnedHostCache.clear();
+    dnsBurnedHostShortCircuitCounter.count = 0;
+}

package/lib/crawler/crawler.d.ts

@@ -71,7 +71,8 @@ export default class Crawler extends EventEmitter<CrawlerEventTypes> {
      * before reaching the dealer so a URL that exists in both sources — which
      * is common in append-mode when a new root coincides with a repromoted
      * previously-external page — does not race on two parallel slots.
-     * @param urls - The list of root URLs to begin crawling from. Must be non-empty.
+     * @param urls - The list of root URLs to begin crawling from. May be empty
+     *   when resumed pending URLs already exist (for example `--retry-failed`).
      * @param opts - Optional overrides; currently only `recursive` is honoured.
      * @param opts.recursive - When `false`, disables recursive discovery and forces list-mode.
      *   Defaults to the constructor option's `recursive` value.
@@ -80,6 +81,7 @@ export default class Crawler extends EventEmitter<CrawlerEventTypes> {
     start(urls: ExURL[], opts?: {
         recursive?: boolean;
     }): void;
+    private _launchBrowserAndScrape;
     /**
      * The default maximum number of concurrent scraping processes.
      *