@nitpicker/crawler 0.14.0 → 0.16.0

package/lib/crawler/assert-chrome-installed.js

@@ -0,0 +1,43 @@
+import { existsSync } from 'node:fs';
+import path from 'node:path';
+import pkg from '../../package.json' with { type: 'json' };
+/**
+ * Verifies that Puppeteer can resolve an installed Chrome/Chromium executable
+ * before a crawl starts.
+ *
+ * A crawl otherwise only discovers a missing browser deep inside the
+ * per-URL scrape loop (`Crawler#_launchBrowserAndScrape`), where it surfaces
+ * as one more scrape error among many — the CLI still prints "Crawl
+ * completed" and writes an archive, so a missing Chrome (a fatal
+ * precondition, not a per-page failure) is easy to miss. Calling this once,
+ * before any archive I/O begins, turns it into an immediate, actionable
+ * failure instead.
+ * @param executablePath - Explicit override, matching
+ *   {@link CrawlerOptions.executablePath}. Pass `null` (or omit) to check
+ *   Puppeteer's own pinned Chrome resolution instead.
+ * @throws {Error} When the resolved executable path does not exist on disk.
+ * @example
+ * ```ts
+ * import { assertChromeIsInstalled } from '@nitpicker/crawler';
+ *
+ * // Throws with install instructions before any crawl work starts.
+ * await assertChromeIsInstalled();
+ * ```
+ */
+export async function assertChromeIsInstalled(executablePath) {
+    if (executablePath) {
+        const execPath = path.resolve(executablePath);
+        if (existsSync(execPath)) {
+            return;
+        }
+        throw new Error(`Executable path does not exist: ${execPath}`);
+    }
+    const puppeteer = await import('puppeteer');
+    const resolvedPath = await puppeteer.executablePath();
+    if (existsSync(resolvedPath)) {
+        return;
+    }
+    const puppeteerVersion = pkg.dependencies.puppeteer;
+    throw new Error(`Chrome executable not found at: ${resolvedPath}\n` +
+        `Run \`npx puppeteer@${puppeteerVersion} browsers install chrome\` to install the Chrome build Puppeteer expects, then retry.`);
+}

package/lib/crawler/choose-probe-host.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Pick the hostname a network-outage probe should target.
+ *
+ * Prefers a host that has already answered successfully during this crawl
+ * session (`Crawler.#successfulHosts` — the same "proven alive" evidence
+ * `shouldBurnHost` uses) over a root URL's hostname, and never falls back to
+ * a hardcoded external address (e.g. `1.1.1.1`): the probe should depend
+ * only on infrastructure the crawl is already touching. `ReadonlySet`
+ * iteration order in JS is insertion order, so this deterministically picks
+ * the first host to have succeeded this session.
+ * @param successfulHosts - Hostnames observed to respond in this session.
+ * @param roots - The crawl's configured root URLs (`CrawlerOptions.roots`),
+ *   used as a fallback before any host has succeeded yet.
+ * @returns A probe target hostname, or `null` if neither source yields one
+ *   (e.g. a fresh session with a malformed/empty roots list and no
+ *   successes yet — the caller should treat this as "cannot probe").
+ * @example
+ * ```ts
+ * chooseProbeHost(new Set(['a.example']), ['https://b.example/']); // 'a.example'
+ * chooseProbeHost(new Set(), ['https://b.example/']); // 'b.example'
+ * chooseProbeHost(new Set(), []); // null
+ * ```
+ */
+export declare function chooseProbeHost(successfulHosts: ReadonlySet<string>, roots: readonly string[]): string | null;

package/lib/crawler/choose-probe-host.js

@@ -0,0 +1,38 @@
+/**
+ * Pick the hostname a network-outage probe should target.
+ *
+ * Prefers a host that has already answered successfully during this crawl
+ * session (`Crawler.#successfulHosts` — the same "proven alive" evidence
+ * `shouldBurnHost` uses) over a root URL's hostname, and never falls back to
+ * a hardcoded external address (e.g. `1.1.1.1`): the probe should depend
+ * only on infrastructure the crawl is already touching. `ReadonlySet`
+ * iteration order in JS is insertion order, so this deterministically picks
+ * the first host to have succeeded this session.
+ * @param successfulHosts - Hostnames observed to respond in this session.
+ * @param roots - The crawl's configured root URLs (`CrawlerOptions.roots`),
+ *   used as a fallback before any host has succeeded yet.
+ * @returns A probe target hostname, or `null` if neither source yields one
+ *   (e.g. a fresh session with a malformed/empty roots list and no
+ *   successes yet — the caller should treat this as "cannot probe").
+ * @example
+ * ```ts
+ * chooseProbeHost(new Set(['a.example']), ['https://b.example/']); // 'a.example'
+ * chooseProbeHost(new Set(), ['https://b.example/']); // 'b.example'
+ * chooseProbeHost(new Set(), []); // null
+ * ```
+ */
+export function chooseProbeHost(successfulHosts, roots) {
+    const [firstSuccessfulHost] = successfulHosts;
+    if (firstSuccessfulHost !== undefined) {
+        return firstSuccessfulHost;
+    }
+    for (const root of roots) {
+        try {
+            return new URL(root).hostname;
+        }
+        catch {
+            continue;
+        }
+    }
+    return null;
+}

package/lib/crawler/clear-dns-burned-host-cache.js

@@ -1,3 +1,4 @@
+import { dnsBurnedHostBurnTimestamps } from './dns-burned-host-burn-timestamps.js';
 import { dnsBurnedHostCache } from './dns-burned-host-cache.js';
 import { dnsBurnedHostShortCircuitCounter } from './dns-burned-host-short-circuit-counter.js';
 /**
@@ -8,4 +9,5 @@ import { dnsBurnedHostShortCircuitCounter } from './dns-burned-host-short-circui
 export function clearDnsBurnedHostCache() {
     dnsBurnedHostCache.clear();
     dnsBurnedHostShortCircuitCounter.count = 0;
+    dnsBurnedHostBurnTimestamps.clear();
 }

package/lib/crawler/crawler.d.ts

@@ -37,6 +37,18 @@ export default class Crawler extends EventEmitter<CrawlerEventTypes> {
      * {@link #runDeal}.
      */
     abort(): void;
+    /**
+     * Per-shape count of anchors the dedupe-cap enqueue gates rejected after
+     * that shape capped (opt-in `--dedupe-cap`). Read by
+     * `CrawlerOrchestrator` at `crawlEnd` to finalize each
+     * `dedupe_cap_events.rejected_count` exactly once — rejections are
+     * accumulated in memory rather than written to the archive per-rejection
+     * to avoid write amplification (a capped trap can generate an unbounded
+     * number of rejected anchors).
+     * @returns A snapshot of the per-shape rejection counts. Empty when
+     *   `--dedupe-cap` was not enabled or no shape has capped yet.
+     */
+    getDedupeCapRejections(): ReadonlyMap<string, number>;
     /**
      * Retrieve the list of Chromium process IDs that are still running.
      *