@nitpicker/crawler 0.15.0 → 0.16.0

package/lib/crawler/types.d.ts

@@ -190,6 +190,29 @@ export interface CrawlerOptions extends Required<Pick<ParseURLOptions, 'disableQ
      * without touching the real network.
      */
     networkProbe: NetworkProbe | null;
+    /**
+     * Same-cluster soft-cap threshold (`--dedupe-cap`), or `null` to disable
+     * the feature entirely (the default). When set, {@link Crawler} stops
+     * enqueueing newly-discovered URLs whose shape (see `computeShapeKey`)
+     * has accumulated this many matching-signature observations (see
+     * `DedupeCapTracker`).
+     */
+    dedupeCap: number | null;
+    /**
+     * Hard cap on the number of distinct URL shapes the same-cluster soft
+     * cap tracks at once (`--dedupe-map-cap`); the least-recently-touched
+     * shape is evicted beyond this. Only relevant when {@link dedupeCap} is
+     * non-null.
+     */
+    dedupeMapCap: number;
+    /**
+     * Shape keys already confirmed capped in a prior session
+     * (persisted as `dedupe_cap_events.shape_key`), seeded into the
+     * tracker's sticky set so `--resume` / `--append` / `--retry-failed` /
+     * `--inventory` do not re-admit a trap this crawl already paid the cost
+     * of discovering once. Ignored when {@link dedupeCap} is `null`.
+     */
+    preloadedStickyShapeKeys: readonly string[];
 }
 /**
  * Inventory-mode runtime configuration. Passed from
@@ -436,6 +459,21 @@ export interface CrawlerEventTypes {
         /** Epoch ms the recovery probe first succeeded. */
         endedAt: number;
     };
+    /**
+     * Emitted the instant the opt-in same-cluster soft cap
+     * ({@link CrawlerOptions.dedupeCap}) confirms a URL shape as a trap (see
+     * `DedupeCapTracker`). The orchestrator persists this via
+     * `Archive.insertDedupeCapEvent` and must remember the returned row id so
+     * `rejected_count` can be finalized once at `crawlEnd` (`Crawler` itself
+     * never touches the archive).
+     */
+    dedupeCap: {
+        shapeKey: string;
+        sampleUrl: string;
+        bodyHash: Buffer;
+        effectiveThreshold: number;
+        observedCount: number;
+    };
 }
 /**
  * Tunables for `NetworkOutageDetector`.

package/lib/crawler-orchestrator.d.ts

@@ -69,6 +69,18 @@ interface CrawlConfig extends Config {
      * `options` so an E2E test can inject it via the public API.
      */
     networkProbe: NetworkProbe | null;
+    /** See {@link CrawlerOptions.dedupeCap}. `null`/omitted disables the feature. */
+    dedupeCap: number | null;
+    /** See {@link CrawlerOptions.dedupeMapCap}. Omitted falls through to `Crawler`'s own default. */
+    dedupeMapCap: number;
+    /**
+     * See {@link CrawlerOptions.preloadedStickyShapeKeys}. Set internally by
+     * the four resuming-session static methods
+     * (`append`/`inventory`/`retryFailed`/`resume`), each independently
+     * calling `archive.listDedupeCapShapeKeys()`; not part of the public
+     * options a caller of those methods passes directly.
+     */
+    preloadedStickyShapeKeys: readonly string[];
 }
 /**
  * Callback invoked after the CrawlerOrchestrator instance is fully initialized

package/lib/crawler-orchestrator.js

@@ -64,6 +64,15 @@ export class CrawlerOrchestrator extends EventEmitter {
     #archive;
     /** The crawler engine that discovers and scrapes pages. */
     #crawler;
+    /**
+     * `dedupe_cap_events.id` for each shape confirmed capped this session, so
+     * `crawlEnd` can look up the right row to finalize with
+     * `Crawler#getDedupeCapRejections`'s counts. A `Map` (not a single
+     * scalar like {@link #openNetworkOutageId}) because, unlike a network
+     * outage, more than one shape can be capped simultaneously within one
+     * crawl.
+     */
+    #dedupeCapEventIds = new Map();
     /** Whether the crawl was started from a pre-defined URL list (non-recursive mode). */
     #fromList;
     /**
@@ -158,6 +167,13 @@ export class CrawlerOrchestrator extends EventEmitter {
             networkOutageHostThreshold: options?.networkOutageHostThreshold,
             networkOutageProbeIntervalMs: options?.networkOutageProbeIntervalMs,
             networkProbe: options?.networkProbe ?? null,
+            dedupeCap: options?.dedupeCap ?? null,
+            dedupeMapCap: options?.dedupeMapCap,
+            // Only the four resuming-session static methods
+            // (`append`/`inventory`/`retryFailed`/`resume`) pass this — a
+            // fresh `crawling()` has no archive history to seed from (see
+            // `CrawlConfig.preloadedStickyShapeKeys`'s JSDoc).
+            preloadedStickyShapeKeys: options?.preloadedStickyShapeKeys ?? [],
         });
     }
     /**
@@ -298,6 +314,23 @@ export class CrawlerOrchestrator extends EventEmitter {
                 })
                     .catch((error) => reject(error));
             });
+            this.#crawler.on('dedupeCap', ({ shapeKey, sampleUrl, bodyHash, effectiveThreshold, observedCount }) => {
+                crawlerLog('Dedupe cap reached: shapeKey=%s effectiveThreshold=%d observedCount=%d', shapeKey, effectiveThreshold, observedCount);
+                console.error(`[dedupe-cap] same-cluster trap confirmed: ${shapeKey} (sample: ${sampleUrl})`);
+                writeQueue
+                    .enqueue(async () => {
+                    const id = await this.#archive.insertDedupeCapEvent({
+                        shapeKey,
+                        sampleUrl,
+                        bodyHash,
+                        effectiveThreshold,
+                        observedCount,
+                        detectedAt: Date.now(),
+                    });
+                    this.#dedupeCapEventIds.set(shapeKey, id);
+                })
+                    .catch((error) => reject(error));
+            });
             this.#crawler.on('response', ({ resource, source }) => {
                 writeQueue
                     .enqueue(() => this.#archive.setResources(resource, source))
@@ -314,6 +347,51 @@ export class CrawlerOrchestrator extends EventEmitter {
                     .catch((error) => reject(error));
             });
             this.#crawler.on('crawlEnd', () => {
+                // Deferred to INSIDE a queued closure, not read synchronously
+                // here, for the same reason `networkOutageRecovered`'s handler
+                // defers reading `#openNetworkOutageId`: a `dedupeCap` event's
+                // INSERT closure may still be queued (not yet executed) at the
+                // instant `crawlEnd` fires. `WriteQueue` runs enqueued
+                // operations in submission order, so by the time THIS closure
+                // executes, every earlier-queued `dedupeCap` INSERT has
+                // already completed and `#dedupeCapEventIds` is reliably
+                // populated.
+                writeQueue
+                    .enqueue(async () => {
+                    const rejections = this.#crawler.getDedupeCapRejections();
+                    // Finalize every shape capped THIS session (has an id in
+                    // `#dedupeCapEventIds`), not just the ones with a nonzero
+                    // rejection count — a shape that capped near the end of the
+                    // crawl (or whose remaining anchors all happened to be
+                    // discovered before it capped) never enters `rejections` at
+                    // all, and would otherwise stay `rejected_count: NULL` forever
+                    // despite the crawl completing normally, corrupting the "NULL
+                    // means the crawl never reached crawlEnd" contract
+                    // `list-dedupe-cap-events.ts` documents.
+                    const shapeKeysToFinalize = new Set([
+                        ...this.#dedupeCapEventIds.keys(),
+                        ...rejections.keys(),
+                    ]);
+                    await Promise.all([...shapeKeysToFinalize].map((shapeKey) => {
+                        const rejectedCount = rejections.get(shapeKey) ?? 0;
+                        const id = this.#dedupeCapEventIds.get(shapeKey);
+                        // A shape capped THIS session has an id here (the
+                        // `dedupeCap` event always enqueues an INSERT before any
+                        // rejection for that shape can be counted) and is
+                        // finalized once via its row id. A shape with no id was
+                        // never observed this session at all — it was preloaded
+                        // into `DedupeCapTracker`'s sticky set from an EARLIER
+                        // session's `dedupe_cap_events` row (see
+                        // `CrawlConfig.preloadedStickyShapeKeys`'s JSDoc), so gate
+                        // rejections still accumulate for it but no new row (and
+                        // thus no id) is ever created. That earlier row's count is
+                        // accumulated onto by shape_key instead of overwritten.
+                        return id === undefined
+                            ? this.#archive.accumulateDedupeCapRejectedCount(shapeKey, rejectedCount)
+                            : this.#archive.finalizeDedupeCapEvent(id, rejectedCount);
+                    }));
+                })
+                    .catch((error) => reject(error));
                 writeQueue
                     .drain()
                     .then(() => resolve())
@@ -521,9 +599,14 @@ export class CrawlerOrchestrator extends EventEmitter {
                     scopeMap.set(parsed.hostname, [...existing, parsed]);
                 }
                 await archive.repromoteExternalPages(scopeMap, archived);
+                // Seed the sticky set from prior sessions' confirmed traps so
+                // `--append` does not pay the cost of re-discovering them (see
+                // `DedupeCapTracker`'s constructor JSDoc).
+                const preloadedStickyShapeKeys = await archive.listDedupeCapShapeKeys();
                 const orchestrator = new CrawlerOrchestrator(archive, {
                     ...mergedConfig,
                     roots: mergedRoots,
+                    preloadedStickyShapeKeys,
                 });
                 const { scraped, pending } = await archive.getCrawlingState();
                 const resources = await archive.getResourceUrlList();
@@ -856,6 +939,16 @@ export class CrawlerOrchestrator extends EventEmitter {
                     inventoryMode: { seedUrls: seedSet },
                 };
                 if (htmlSeeds.length > 0) {
+                    // Seed the sticky set from prior sessions' confirmed traps
+                    // so `--inventory` does not pay the cost of
+                    // re-discovering them (see `DedupeCapTracker`'s
+                    // constructor JSDoc). Scoped to this branch only,
+                    // matching `#preloadDnsBurnedHostCache`'s scoping below —
+                    // the fallback (non-HTML-only) branch never calls
+                    // `orchestrator.crawling(...)`, so the tracker is never
+                    // consulted there.
+                    orchestratorOptions.preloadedStickyShapeKeys =
+                        await archive.listDedupeCapShapeKeys();
                     const orchestrator = new CrawlerOrchestrator(archive, orchestratorOptions);
                     // Re-read pending *after* the pre-insert so the strict-
                     // pending set includes the freshly inserted
@@ -1007,7 +1100,14 @@ export class CrawlerOrchestrator extends EventEmitter {
                 log('Start retrying failed pages');
                 log('Archive %s', absFilePath);
                 log('Reset %d failed page(s)', resetUrls.length);
-                const orchestrator = new CrawlerOrchestrator(archive, config);
+                // Seed the sticky set from prior sessions' confirmed traps so
+                // `--retry-failed` does not pay the cost of re-discovering
+                // them (see `DedupeCapTracker`'s constructor JSDoc).
+                const preloadedStickyShapeKeys = await archive.listDedupeCapShapeKeys();
+                const orchestrator = new CrawlerOrchestrator(archive, {
+                    ...config,
+                    preloadedStickyShapeKeys,
+                });
                 const { scraped, pending } = await archive.getCrawlingState();
                 const resources = await archive.getResourceUrlList();
                 const pagesScrapedOffset = await archive.getScrapedHtmlPageCount();
@@ -1056,9 +1156,14 @@ export class CrawlerOrchestrator extends EventEmitter {
     static async resume(stubPath, options, initializedCallback) {
         const archive = await Archive.resume(stubPath);
         const archivedConfig = await archive.getConfig();
+        // Seed the sticky set from prior sessions' confirmed traps so
+        // `--resume` does not pay the cost of re-discovering them (see
+        // `DedupeCapTracker`'s constructor JSDoc).
+        const preloadedStickyShapeKeys = await archive.listDedupeCapShapeKeys();
         const config = {
             ...archivedConfig,
             ...cleanObject(options),
+            preloadedStickyShapeKeys,
         };
         const orchestrator = new CrawlerOrchestrator(archive, config);
         const _url = await archive.getUrl();

package/lib/crawler.d.ts

@@ -46,6 +46,7 @@ export type { NetworkProbe } from './crawler/probe-network.js';
 export { probeNetwork } from './crawler/probe-network.js';
 export { computeOutageClampTimestamp } from './archive/db-ops/outages/compute-outage-clamp-timestamp.js';
 export { chooseProbeHost } from './crawler/choose-probe-host.js';
+export { assertChromeIsInstalled } from './crawler/assert-chrome-installed.js';
 export { computeFileSha256 } from './utils/compute-file-sha256.js';
 export { populateEntityTables } from './archive/populate-entity-tables/populate-entities.js';
 export type { PageDomPathResolver } from './archive/populate-entity-tables/populate-image-items.js';

package/lib/crawler.js

@@ -43,6 +43,7 @@ export { default as NetworkGate } from './crawler/network-gate.js';
 export { probeNetwork } from './crawler/probe-network.js';
 export { computeOutageClampTimestamp } from './archive/db-ops/outages/compute-outage-clamp-timestamp.js';
 export { chooseProbeHost } from './crawler/choose-probe-host.js';
+export { assertChromeIsInstalled } from './crawler/assert-chrome-installed.js';
 export { computeFileSha256 } from './utils/compute-file-sha256.js';
 // 0.13 ref-table population (issue #191, epic #103). Exposed as the
 // public seam that the migration script (`scripts/migrate-to-0.13.mjs`)

package/lib/permanent-error-kinds.d.ts

@@ -7,11 +7,12 @@ import type { ErrorKind } from './types.js';
  * Used by `resetFailedPages` to exclude pages whose latest recorded error
  * falls in this set, so `--retry-failed` actually converges: without the
  * exclusion, NXDOMAIN / TLS mismatch / `ERR_BLOCKED_BY_CLIENT` /
- * `ECONNREFUSED` / HTTP parse-error pages would be reset to pending on every
- * iteration, the crawler would re-attempt them, they would fail again the
- * same way, and the retry-target count would stay constant forever.
+ * `ECONNREFUSED` / HTTP parse-error / redirect-loop pages would be reset to
+ * pending on every iteration, the crawler would re-attempt them, they would
+ * fail again the same way, and the retry-target count would stay constant
+ * forever.
  *
- * Why these five and not others:
+ * Why these six and not others:
  * - **dns** — `ENOTFOUND` / `ERR_NAME_NOT_RESOLVED` are authoritative DNS
  *   answers; the host is gone (or never existed). EAI_AGAIN is split out as
  *   `dns-transient` precisely so it is NOT in this set.
@@ -28,6 +29,10 @@ import type { ErrorKind } from './types.js';
  *   the listener; either no process is listening on the port or its accept
  *   queue rejected the connection. Either way the answer is final until the
  *   server operator intervenes.
+ * - **redirect-loop** — `Maximum number of redirects exceeded` /
+ *   `ERR_TOO_MANY_REDIRECTS` means the site's own redirect chain never
+ *   terminates; the exact same chain is served on every future fetch until
+ *   the site operator fixes it.
  *
  * Notably absent (intentionally retryable):
  * - `connection-reset` / `connection-timeout` — could be middlebox or

package/lib/permanent-error-kinds.js

@@ -6,11 +6,12 @@
  * Used by `resetFailedPages` to exclude pages whose latest recorded error
  * falls in this set, so `--retry-failed` actually converges: without the
  * exclusion, NXDOMAIN / TLS mismatch / `ERR_BLOCKED_BY_CLIENT` /
- * `ECONNREFUSED` / HTTP parse-error pages would be reset to pending on every
- * iteration, the crawler would re-attempt them, they would fail again the
- * same way, and the retry-target count would stay constant forever.
+ * `ECONNREFUSED` / HTTP parse-error / redirect-loop pages would be reset to
+ * pending on every iteration, the crawler would re-attempt them, they would
+ * fail again the same way, and the retry-target count would stay constant
+ * forever.
  *
- * Why these five and not others:
+ * Why these six and not others:
  * - **dns** — `ENOTFOUND` / `ERR_NAME_NOT_RESOLVED` are authoritative DNS
  *   answers; the host is gone (or never existed). EAI_AGAIN is split out as
  *   `dns-transient` precisely so it is NOT in this set.
@@ -27,6 +28,10 @@
  *   the listener; either no process is listening on the port or its accept
  *   queue rejected the connection. Either way the answer is final until the
  *   server operator intervenes.
+ * - **redirect-loop** — `Maximum number of redirects exceeded` /
+ *   `ERR_TOO_MANY_REDIRECTS` means the site's own redirect chain never
+ *   terminates; the exact same chain is served on every future fetch until
+ *   the site operator fixes it.
  *
  * Notably absent (intentionally retryable):
  * - `connection-reset` / `connection-timeout` — could be middlebox or
@@ -45,4 +50,5 @@ export const PERMANENT_ERROR_KINDS = new Set([
     'client-blocked',
     'parse-error',
     'connection-refused',
+    'redirect-loop',
 ]);

package/lib/types.d.ts

@@ -66,6 +66,7 @@ export interface InventoryRunAggregates {
  * | `local-network` | **yes** | no | local machine's network is unreachable / changed (WiFi, sleep, ICMP-unreachable, …) |
  * | `parse-error` | mostly persistent | no | HTTP response could not be parsed (proxy, garbage, MITM) |
  * | `client-blocked` | persistent (per browser) | no | Chromium-side `ERR_BLOCKED_BY_*` family — the browser actively refused the request (ad/tracker heuristics, CSP, CORP, administrator block list, …) |
+ * | `redirect-loop` | no | no | the redirect chain exceeded `follow-redirects`' `maxRedirects` limit — the site's own redirect configuration never converges |
  * | `protocol` | yes | no | puppeteer protocol layer (frame detached, target closed, …) |
  * | `timeout` | yes | no | puppeteer navigation timeout or HEAD pre-flight race timeout (`Timeout: <url>`) |
  * | `unknown` | unknown | no | catch-all for messages no matcher recognised |
@@ -90,7 +91,7 @@ export interface InventoryRunAggregates {
  * entry), or a recoverable URL never reaching the puppeteer fallback (no
  * PUPPETEER_FALLBACK entry).
  */
-export type ErrorKind = 'dns' | 'dns-transient' | 'connection-refused' | 'connection-reset' | 'connection-timeout' | 'tls' | 'local-network' | 'parse-error' | 'client-blocked' | 'timeout' | 'protocol' | 'unknown';
+export type ErrorKind = 'dns' | 'dns-transient' | 'connection-refused' | 'connection-reset' | 'connection-timeout' | 'tls' | 'local-network' | 'parse-error' | 'client-blocked' | 'redirect-loop' | 'timeout' | 'protocol' | 'unknown';
 /**
  * Event map for the `CrawlerOrchestrator` class.
  *

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@nitpicker/crawler",
-	"version": "0.15.0",
+	"version": "0.16.0",
 	"description": "Web crawler engine with headless browser rendering and archive storage",
 	"author": "D-ZERO",
 	"license": "Apache-2.0",
@@ -48,5 +48,5 @@
 		"@types/tar": "7.0.87",
 		"@types/unzipper": "0.10.11"
 	},
-	"gitHead": "d1485df6e43375c6a44edfe07a0293fd39d7810d"
+	"gitHead": "bef8b6d48e3ca5167fee643d6aba8644a065df4a"
 }