@nitpicker/crawler 0.15.0 → 0.16.0

package/lib/archive/archive.d.ts

@@ -1,5 +1,5 @@
 import type { TemplateClusterReason } from './db-ops/analysis/types.js';
-import type { Config, InsertNetworkOutageParams, InventoryRunMeta, PageSource } from './types.js';
+import type { Config, InsertDedupeCapEventParams, InsertNetworkOutageParams, InventoryRunMeta, PageSource } from './types.js';
 import type { OutageWindow } from '../is-within-outage-window.js';
 import type { PageData, CrawlerError, Resource } from '../utils/types/types.js';
 import type { ConsoleLogEntry } from '@d-zero/beholder';
@@ -49,6 +49,17 @@ export default class Archive extends ArchiveAccessor {
      * @deprecated This method is no longer functional.
      */
     abort(): void;
+    /**
+     * Adds onto the `rejected_count` of a shape's `dedupe_cap_events` row,
+     * looked up by `shape_key` rather than `id` — used for a shape that
+     * capped in an earlier session (preloaded into `DedupeCapTracker`'s
+     * sticky set) and so has no event id from the current session.
+     *
+     * Thin facade over {@link Database.accumulateDedupeCapRejectedCount}.
+     * @param shapeKey - The capped shape whose rejection count to accumulate.
+     * @param rejectedCount - Additional anchors rejected for this shape in the current session.
+     */
+    accumulateDedupeCapRejectedCount(shapeKey: string, rejectedCount: number): Promise<void>;
     /**
      * Records a crawler-level error to both the human-readable `error.log` (full
      * stack, for debugging) and the structured `crawl_errors` table (queryable,
@@ -80,6 +91,15 @@ export default class Archive extends ArchiveAccessor {
      * @param endedAt - Epoch ms the outage is considered to have ended.
      */
     closeNetworkOutage(id: number, endedAt: number): Promise<void>;
+    /**
+     * Finalizes a `dedupe_cap_events` row by stamping `rejected_count` — a
+     * no-op if already finalized.
+     *
+     * Thin facade over {@link Database.finalizeDedupeCapEvent}.
+     * @param id - The `dedupe_cap_events.id` to finalize.
+     * @param rejectedCount - Number of anchors rejected for this shape after it capped.
+     */
+    finalizeDedupeCapEvent(id: number, rejectedCount: number): Promise<void>;
     /**
      * Retrieves the current crawling state, including lists of scraped and pending URLs.
      * @returns An object with `scraped` and `pending` URL arrays.
@@ -136,6 +156,15 @@ export default class Archive extends ArchiveAccessor {
      * @returns The base URL string.
      */
     getUrl(): Promise<string>;
+    /**
+     * Appends one row (`rejected_count = NULL`) to the `dedupe_cap_events`
+     * journal.
+     *
+     * Thin facade over {@link Database.insertDedupeCapEvent}.
+     * @param params - The newly-capped shape's fields to record.
+     * @returns The autoincremented `id` of the inserted row.
+     */
+    insertDedupeCapEvent(params: InsertDedupeCapEventParams): Promise<number>;
     /**
      * Pre-insert inventory non-HTML URLs as `source='inventory-seed'`
      * placeholders in the `resources` table — the non-HTML counterpart of
@@ -174,6 +203,15 @@ export default class Archive extends ArchiveAccessor {
      * @returns The autoincremented `id` of the inserted row.
      */
     insertNetworkOutage(params: InsertNetworkOutageParams): Promise<number>;
+    /**
+     * Every distinct `dedupe_cap_events.shape_key` recorded in this archive.
+     * Consumed by `CrawlerOrchestrator` to preload `DedupeCapTracker`'s
+     * sticky set on `--resume` / `--append` / `--retry-failed` /
+     * `--inventory`, mirroring {@link listDnsBurnedHostCandidates}'s
+     * writer-only exposure.
+     * @returns Distinct shape keys already confirmed capped.
+     */
+    listDedupeCapShapeKeys(): Promise<string[]>;
     /**
      * Hostnames whose `crawl_errors` history is consistently DNS failures and
      * for which no recent 2xx/3xx page or resource is recorded. Consumed by

package/lib/archive/archive.js

@@ -94,6 +94,20 @@ export default class Archive extends ArchiveAccessor {
      * @deprecated This method is no longer functional.
      */
     abort() { }
+    /**
+     * Adds onto the `rejected_count` of a shape's `dedupe_cap_events` row,
+     * looked up by `shape_key` rather than `id` — used for a shape that
+     * capped in an earlier session (preloaded into `DedupeCapTracker`'s
+     * sticky set) and so has no event id from the current session.
+     *
+     * Thin facade over {@link Database.accumulateDedupeCapRejectedCount}.
+     * @param shapeKey - The capped shape whose rejection count to accumulate.
+     * @param rejectedCount - Additional anchors rejected for this shape in the current session.
+     */
+    async accumulateDedupeCapRejectedCount(shapeKey, rejectedCount) {
+        dbLog('Accumulate dedupe cap rejected count shapeKey=%s rejectedCount=%d', shapeKey, rejectedCount);
+        return await this.#db.accumulateDedupeCapRejectedCount(shapeKey, rejectedCount);
+    }
     /**
      * Records a crawler-level error to both the human-readable `error.log` (full
      * stack, for debugging) and the structured `crawl_errors` table (queryable,
@@ -135,6 +149,18 @@ export default class Archive extends ArchiveAccessor {
         dbLog('Close network outage id=%d endedAt=%d', id, endedAt);
         return await this.#db.closeNetworkOutage(id, endedAt);
     }
+    /**
+     * Finalizes a `dedupe_cap_events` row by stamping `rejected_count` — a
+     * no-op if already finalized.
+     *
+     * Thin facade over {@link Database.finalizeDedupeCapEvent}.
+     * @param id - The `dedupe_cap_events.id` to finalize.
+     * @param rejectedCount - Number of anchors rejected for this shape after it capped.
+     */
+    async finalizeDedupeCapEvent(id, rejectedCount) {
+        dbLog('Finalize dedupe cap event id=%d rejectedCount=%d', id, rejectedCount);
+        return await this.#db.finalizeDedupeCapEvent(id, rejectedCount);
+    }
     /**
      * Retrieves the current crawling state, including lists of scraped and pending URLs.
      * @returns An object with `scraped` and `pending` URL arrays.
@@ -202,6 +228,18 @@ export default class Archive extends ArchiveAccessor {
     async getUrl() {
         return this.#db.getBaseUrl();
     }
+    /**
+     * Appends one row (`rejected_count = NULL`) to the `dedupe_cap_events`
+     * journal.
+     *
+     * Thin facade over {@link Database.insertDedupeCapEvent}.
+     * @param params - The newly-capped shape's fields to record.
+     * @returns The autoincremented `id` of the inserted row.
+     */
+    async insertDedupeCapEvent(params) {
+        dbLog('Insert dedupe cap event: shapeKey=%s', params.shapeKey);
+        return await this.#db.insertDedupeCapEvent(params);
+    }
     /**
      * Pre-insert inventory non-HTML URLs as `source='inventory-seed'`
      * placeholders in the `resources` table — the non-HTML counterpart of
@@ -255,6 +293,17 @@ export default class Archive extends ArchiveAccessor {
         dbLog('Insert network outage: startedAt=%d probeHost=%s', params.startedAt, params.probeHost);
         return await this.#db.insertNetworkOutage(params);
     }
+    /**
+     * Every distinct `dedupe_cap_events.shape_key` recorded in this archive.
+     * Consumed by `CrawlerOrchestrator` to preload `DedupeCapTracker`'s
+     * sticky set on `--resume` / `--append` / `--retry-failed` /
+     * `--inventory`, mirroring {@link listDnsBurnedHostCandidates}'s
+     * writer-only exposure.
+     * @returns Distinct shape keys already confirmed capped.
+     */
+    async listDedupeCapShapeKeys() {
+        return this.#db.listDedupeCapShapeKeys();
+    }
     /**
      * Hostnames whose `crawl_errors` history is consistently DNS failures and
      * for which no recent 2xx/3xx page or resource is recorded. Consumed by

package/lib/archive/create-adjunct-tables.d.ts

@@ -16,6 +16,9 @@ import type { Knex } from 'knex';
  * - `inventory_runs` — `--inventory` audit log (no FK; append-only)
  * - `network_outages` — operator-network-outage journal (no FK; append-only
  *   except `ended_at`, which is written once on recovery)
+ * - `dedupe_cap_events` — `--dedupe-cap` same-cluster soft-cap audit log (no
+ *   FK; append-only except `rejected_count`, which is written once at
+ *   `crawlEnd`)
  * - `analysis_text_refs` + `analysis_violations` — analyze-phase findings,
  *   FK → `content_items(id)`
  * - `page_templates` — DOM-structure template classification (`--templates`,

package/lib/archive/create-adjunct-tables.js

@@ -15,6 +15,9 @@
  * - `inventory_runs` — `--inventory` audit log (no FK; append-only)
  * - `network_outages` — operator-network-outage journal (no FK; append-only
  *   except `ended_at`, which is written once on recovery)
+ * - `dedupe_cap_events` — `--dedupe-cap` same-cluster soft-cap audit log (no
+ *   FK; append-only except `rejected_count`, which is written once at
+ *   `crawlEnd`)
  * - `analysis_text_refs` + `analysis_violations` — analyze-phase findings,
  *   FK → `content_items(id)`
  * - `page_templates` — DOM-structure template classification (`--templates`,
@@ -326,6 +329,45 @@ export async function createAdjunctTables(instance) {
             t.integer('trigger_host_count').notNullable();
         });
     }
+    if (!(await instance.schema.hasTable('dedupe_cap_events'))) {
+        await instance.schema.createTable('dedupe_cap_events', (t) => {
+            // One row per URL shape the `--dedupe-cap` same-cluster soft cap
+            // (`DedupeCapTracker`) confirmed as a trap during this crawl. No
+            // index: a crawl produces at most a handful of these rows (same
+            // reasoning as `network_outages`, above).
+            t.increments('id');
+            // The URL shape key (`computeShapeKey`) that capped — a template
+            // with placeholders (e.g. `example.com/news/date/{n}/`), not a
+            // literal URL.
+            t.string('shape_key').notNullable();
+            // One concrete URL matching this shape, captured at cap time so a
+            // human reading the audit log can identify what was being
+            // crawled — `shape_key` alone is a template, not a navigable URL.
+            t.string('sample_url').notNullable();
+            // `computeBodyHash` result recorded at cap time. Nullable only in
+            // the sense that BLOB columns are nullable by default; every row
+            // this feature writes populates it (a page with no rendered
+            // `<body>` never reaches the tracker — see `Crawler#handleResult`).
+            t.binary('body_hash').nullable();
+            // The Misra-Gries threshold that actually triggered the cap,
+            // after halving for the `body_hash`-match / `og:url`-mismatch
+            // confidence signals — NOT necessarily equal to `--dedupe-cap`'s
+            // raw value.
+            t.integer('effective_threshold').notNullable();
+            // The tracker's Misra-Gries counter value at cap time: a LOWER
+            // BOUND on the number of matching-signature pages seen for this
+            // shape, not an exact observation count (see `DedupeCapTracker`).
+            t.integer('observed_count').notNullable();
+            t.integer('detected_at').notNullable();
+            // NULL until `crawlEnd` finalizes it (see
+            // `Crawler#getDedupeCapRejections`). Unlike `network_outages.ended_at`,
+            // a NULL here has no ambiguous "still ongoing" reading — a
+            // crawl that never reached `crawlEnd` simply left the count
+            // undetermined, so no boot-time reconciliation pass is needed
+            // (readers display "unknown", not "0" or "unbounded").
+            t.integer('rejected_count').nullable();
+        });
+    }
     if (!(await instance.schema.hasTable('analysis_text_refs'))) {
         await instance.raw(`
 			CREATE TABLE analysis_text_refs (

package/lib/archive/database.d.ts

@@ -1,6 +1,6 @@
 import type { TemplateClusterReason } from './db-ops/analysis/types.js';
 import type { JsonLdRow, MainContentAudioRow, MainContentButtonRow, MainContentCanvasRow, MainContentHeadingRow, MainContentIframeRow, MainContentImageRow, MainContentTableRow, MainContentVideoRow, TagRow } from './meta/types.js';
-import type { Config, DatabaseOption, DB_Redirect, DB_Resource, DatabaseEvent, InsertNetworkOutageParams, InventoryRunMeta, PageFilter, PageSource } from './types.js';
+import type { Config, DatabaseOption, DB_Redirect, DB_Resource, DatabaseEvent, InsertDedupeCapEventParams, InsertNetworkOutageParams, InventoryRunMeta, PageFilter, PageSource } from './types.js';
 import type { OutageWindow } from '../is-within-outage-window.js';
 import type { PageData, Resource } from '../utils/types/types.js';
 import type { ConsoleLogEntry } from '@d-zero/beholder';
@@ -32,6 +32,14 @@ import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-awa
 export declare class Database extends EventEmitter<DatabaseEvent> {
     #private;
     private constructor();
+    /**
+     * Adds onto the `rejected_count` of a shape's `dedupe_cap_events` row,
+     * looked up by `shape_key` rather than `id`. Delegates to
+     * {@link accumulateDedupeCapRejectedCountOp}.
+     * @param shapeKey - The capped shape whose rejection count to accumulate.
+     * @param rejectedCount - Additional anchors rejected for this shape in the current session.
+     */
+    accumulateDedupeCapRejectedCount(shapeKey: string, rejectedCount: number): Promise<void>;
     /**
      * Forces a WAL checkpoint, writing all pending WAL data back to the main
      * database file. Delegates to {@link checkpointOp}.
@@ -49,6 +57,14 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * Delegates to {@link destroyOp}.
      */
     destroy(): Promise<void>;
+    /**
+     * Finalizes a `dedupe_cap_events` row by stamping `rejected_count` — a
+     * no-op if the row is already finalized. Delegates to
+     * {@link finalizeDedupeCapEventOp}.
+     * @param id - The `dedupe_cap_events.id` to finalize.
+     * @param rejectedCount - Number of anchors rejected for this shape after it capped.
+     */
+    finalizeDedupeCapEvent(id: number, rejectedCount: number): Promise<void>;
     /**
      * Retrieves all anchors (outgoing links) on a specific page.
      * Delegates to {@link getAnchorsOnPageOp}.
@@ -273,6 +289,13 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @param isExternal - Whether the URL is external to the crawl scope.
      */
     insertCrawlError(url: string | null, message: string, isExternal?: boolean): Promise<void>;
+    /**
+     * Appends one row (`rejected_count = NULL`) to the `dedupe_cap_events`
+     * journal. Delegates to {@link insertDedupeCapEventOp}.
+     * @param params - The newly-capped shape's fields to record.
+     * @returns The autoincremented `id` of the newly-inserted row.
+     */
+    insertDedupeCapEvent(params: InsertDedupeCapEventParams): Promise<number>;
     /**
      * Pre-insert inventory non-HTML URLs into `resources` as placeholder rows.
      * Delegates to {@link insertInventoryResourcesOp}.
@@ -317,6 +340,13 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @param pageUrl - The URL of the page that references the resource.
      */
     insertResourceReferrers(src: string, pageUrl: string): Promise<void>;
+    /**
+     * Every distinct `dedupe_cap_events.shape_key` recorded in this archive.
+     * Delegates to {@link listDedupeCapShapeKeysOp}.
+     * @returns Distinct shape keys already confirmed capped, or `[]` on an
+     *   archive that predates `dedupe_cap_events` or has recorded none.
+     */
+    listDedupeCapShapeKeys(): Promise<string[]>;
     /**
      * Hostnames whose `crawl_errors` history is consistently DNS failures and
      * for which no recent 2xx-3xx page or resource is recorded.

package/lib/archive/database.js

@@ -16,6 +16,10 @@ import { getName as getNameOp } from './db-ops/config/get-name.js';
 import { setConfig as setConfigOp } from './db-ops/config/set-config.js';
 import { updateConfig as updateConfigOp } from './db-ops/config/update-config.js';
 import { replaceConsoleLogs as replaceConsoleLogsOp } from './db-ops/console-logs/replace-console-logs.js';
+import { accumulateDedupeCapRejectedCount as accumulateDedupeCapRejectedCountOp } from './db-ops/dedupe-cap/accumulate-dedupe-cap-rejected-count.js';
+import { finalizeDedupeCapEvent as finalizeDedupeCapEventOp } from './db-ops/dedupe-cap/finalize-dedupe-cap-event.js';
+import { insertDedupeCapEvent as insertDedupeCapEventOp } from './db-ops/dedupe-cap/insert-dedupe-cap-event.js';
+import { listDedupeCapShapeKeys as listDedupeCapShapeKeysOp } from './db-ops/dedupe-cap/list-dedupe-cap-shape-keys.js';
 import { insertCrawlError as insertCrawlErrorOp } from './db-ops/errors/insert-crawl-error.js';
 import { insertPageError as insertPageErrorOp } from './db-ops/errors/insert-page-error.js';
 import { listDnsBurnedHostCandidates as listDnsBurnedHostCandidatesOp } from './db-ops/errors/list-dns-burned-host-candidates.js';
@@ -118,6 +122,16 @@ export class Database extends EventEmitter {
             },
         });
     }
+    /**
+     * Adds onto the `rejected_count` of a shape's `dedupe_cap_events` row,
+     * looked up by `shape_key` rather than `id`. Delegates to
+     * {@link accumulateDedupeCapRejectedCountOp}.
+     * @param shapeKey - The capped shape whose rejection count to accumulate.
+     * @param rejectedCount - Additional anchors rejected for this shape in the current session.
+     */
+    async accumulateDedupeCapRejectedCount(shapeKey, rejectedCount) {
+        return emitErrorAndRetry(this, 'Database.accumulateDedupeCapRejectedCount', async () => await accumulateDedupeCapRejectedCountOp(this.#instance, shapeKey, rejectedCount), retrySetting);
+    }
     /**
      * Forces a WAL checkpoint, writing all pending WAL data back to the main
      * database file. Delegates to {@link checkpointOp}.
@@ -141,6 +155,16 @@ export class Database extends EventEmitter {
     async destroy() {
         await destroyOp(this.#instance);
     }
+    /**
+     * Finalizes a `dedupe_cap_events` row by stamping `rejected_count` — a
+     * no-op if the row is already finalized. Delegates to
+     * {@link finalizeDedupeCapEventOp}.
+     * @param id - The `dedupe_cap_events.id` to finalize.
+     * @param rejectedCount - Number of anchors rejected for this shape after it capped.
+     */
+    async finalizeDedupeCapEvent(id, rejectedCount) {
+        return emitErrorAndRetry(this, 'Database.finalizeDedupeCapEvent', async () => await finalizeDedupeCapEventOp(this.#instance, id, rejectedCount), retrySetting);
+    }
     /**
      * Retrieves all anchors (outgoing links) on a specific page.
      * Delegates to {@link getAnchorsOnPageOp}.
@@ -422,6 +446,15 @@ export class Database extends EventEmitter {
     async insertCrawlError(url, message, isExternal = false) {
         return emitErrorAndRetry(this, 'Database.insertCrawlError', async () => await insertCrawlErrorOp(this.#instance, url, message, isExternal), retrySetting);
     }
+    /**
+     * Appends one row (`rejected_count = NULL`) to the `dedupe_cap_events`
+     * journal. Delegates to {@link insertDedupeCapEventOp}.
+     * @param params - The newly-capped shape's fields to record.
+     * @returns The autoincremented `id` of the newly-inserted row.
+     */
+    async insertDedupeCapEvent(params) {
+        return emitErrorAndRetry(this, 'Database.insertDedupeCapEvent', async () => await insertDedupeCapEventOp(this.#instance, params), retrySetting);
+    }
     /**
      * Pre-insert inventory non-HTML URLs into `resources` as placeholder rows.
      * Delegates to {@link insertInventoryResourcesOp}.
@@ -478,6 +511,15 @@ export class Database extends EventEmitter {
     async insertResourceReferrers(src, pageUrl) {
         return emitErrorAndRetry(this, 'Database.insertResourceReferrers', async () => await insertResourceReferrersOp(this.#instance, this.#writeRefCaches, src, pageUrl), retrySetting);
     }
+    /**
+     * Every distinct `dedupe_cap_events.shape_key` recorded in this archive.
+     * Delegates to {@link listDedupeCapShapeKeysOp}.
+     * @returns Distinct shape keys already confirmed capped, or `[]` on an
+     *   archive that predates `dedupe_cap_events` or has recorded none.
+     */
+    async listDedupeCapShapeKeys() {
+        return emitErrorAndRetry(this, 'Database.listDedupeCapShapeKeys', async () => await listDedupeCapShapeKeysOp(this.#instance), retrySetting);
+    }
     /**
      * Hostnames whose `crawl_errors` history is consistently DNS failures and
      * for which no recent 2xx-3xx page or resource is recorded.

package/lib/archive/db-ops/dedupe-cap/accumulate-dedupe-cap-rejected-count.d.ts

@@ -0,0 +1,18 @@
+import type { Knex } from 'knex';
+/**
+ * Adds `rejectedCount` onto the `rejected_count` of the `dedupe_cap_events`
+ * row for `shapeKey`, treating a still-`NULL` count as `0`. Unlike
+ * `finalize-dedupe-cap-event.ts` (which stamps a session's own newly-capped
+ * shape exactly once, guarded by `whereNull`), this targets a shape that
+ * capped in an EARLIER session and was preloaded into `DedupeCapTracker`'s
+ * sticky set (see `DedupeCapTracker`'s constructor JSDoc) — gate rejections
+ * for such a shape still occur in the current session, but no `dedupeCap`
+ * event (and thus no new row) is ever emitted for it, since the tracker
+ * short-circuits on an already-sticky shape before `observe` runs. Matches
+ * by `shape_key` rather than `id` because the caller (`CrawlerOrchestrator`)
+ * only has the shape key for a preloaded-sticky shape, never its row id.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param shapeKey - The capped shape whose rejection count to accumulate.
+ * @param rejectedCount - Additional anchors rejected for this shape in the current session.
+ */
+export declare function accumulateDedupeCapRejectedCount(knex: Knex, shapeKey: string, rejectedCount: number): Promise<void>;

package/lib/archive/db-ops/dedupe-cap/accumulate-dedupe-cap-rejected-count.js

@@ -0,0 +1,23 @@
+/**
+ * Adds `rejectedCount` onto the `rejected_count` of the `dedupe_cap_events`
+ * row for `shapeKey`, treating a still-`NULL` count as `0`. Unlike
+ * `finalize-dedupe-cap-event.ts` (which stamps a session's own newly-capped
+ * shape exactly once, guarded by `whereNull`), this targets a shape that
+ * capped in an EARLIER session and was preloaded into `DedupeCapTracker`'s
+ * sticky set (see `DedupeCapTracker`'s constructor JSDoc) — gate rejections
+ * for such a shape still occur in the current session, but no `dedupeCap`
+ * event (and thus no new row) is ever emitted for it, since the tracker
+ * short-circuits on an already-sticky shape before `observe` runs. Matches
+ * by `shape_key` rather than `id` because the caller (`CrawlerOrchestrator`)
+ * only has the shape key for a preloaded-sticky shape, never its row id.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param shapeKey - The capped shape whose rejection count to accumulate.
+ * @param rejectedCount - Additional anchors rejected for this shape in the current session.
+ */
+export async function accumulateDedupeCapRejectedCount(knex, shapeKey, rejectedCount) {
+    await knex('dedupe_cap_events')
+        .where({ shape_key: shapeKey })
+        .update({
+        rejected_count: knex.raw('COALESCE(rejected_count, 0) + ?', [rejectedCount]),
+    });
+}

package/lib/archive/db-ops/dedupe-cap/finalize-dedupe-cap-event.d.ts

@@ -0,0 +1,12 @@
+import type { Knex } from 'knex';
+/**
+ * Finalizes a `dedupe_cap_events` row by stamping `rejected_count` — but
+ * ONLY if it is still unset. The `whereNull('rejected_count')` guard makes
+ * this idempotent, mirroring `close-network-outage.ts`'s `ended_at` guard: a
+ * second call matches zero rows and is a silent no-op rather than
+ * overwriting an already-finalized count.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param id - The `dedupe_cap_events.id` to finalize.
+ * @param rejectedCount - Number of anchors rejected for this shape after it capped.
+ */
+export declare function finalizeDedupeCapEvent(knex: Knex, id: number, rejectedCount: number): Promise<void>;

package/lib/archive/db-ops/dedupe-cap/finalize-dedupe-cap-event.js

@@ -0,0 +1,15 @@
+/**
+ * Finalizes a `dedupe_cap_events` row by stamping `rejected_count` — but
+ * ONLY if it is still unset. The `whereNull('rejected_count')` guard makes
+ * this idempotent, mirroring `close-network-outage.ts`'s `ended_at` guard: a
+ * second call matches zero rows and is a silent no-op rather than
+ * overwriting an already-finalized count.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param id - The `dedupe_cap_events.id` to finalize.
+ * @param rejectedCount - Number of anchors rejected for this shape after it capped.
+ */
+export async function finalizeDedupeCapEvent(knex, id, rejectedCount) {
+    await knex('dedupe_cap_events').where({ id }).whereNull('rejected_count').update({
+        rejected_count: rejectedCount,
+    });
+}

package/lib/archive/db-ops/dedupe-cap/insert-dedupe-cap-event.d.ts

@@ -0,0 +1,14 @@
+import type { InsertDedupeCapEventParams } from '../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Appends one row to the `dedupe_cap_events` journal, with `rejected_count`
+ * left `NULL` — the row starts life without a finalized rejection count.
+ *
+ * Called the instant `DedupeCapTracker#observe` confirms a URL shape as a
+ * same-cluster trap (the `dedupeCap` event). See
+ * `finalize-dedupe-cap-event.ts` for how `rejected_count` is later set.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param params - The newly-capped shape's fields to record.
+ * @returns The autoincremented `id` of the newly-inserted row.
+ */
+export declare function insertDedupeCapEvent(knex: Knex, params: InsertDedupeCapEventParams): Promise<number>;

package/lib/archive/db-ops/dedupe-cap/insert-dedupe-cap-event.js

@@ -0,0 +1,30 @@
+/**
+ * Appends one row to the `dedupe_cap_events` journal, with `rejected_count`
+ * left `NULL` — the row starts life without a finalized rejection count.
+ *
+ * Called the instant `DedupeCapTracker#observe` confirms a URL shape as a
+ * same-cluster trap (the `dedupeCap` event). See
+ * `finalize-dedupe-cap-event.ts` for how `rejected_count` is later set.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param params - The newly-capped shape's fields to record.
+ * @returns The autoincremented `id` of the newly-inserted row.
+ */
+export async function insertDedupeCapEvent(knex, params) {
+    const inserted = await knex
+        .from('dedupe_cap_events')
+        .insert({
+        shape_key: params.shapeKey,
+        sample_url: params.sampleUrl,
+        body_hash: params.bodyHash,
+        effective_threshold: params.effectiveThreshold,
+        observed_count: params.observedCount,
+        detected_at: params.detectedAt,
+        rejected_count: null,
+    })
+        .returning('id');
+    const id = inserted[0]?.id;
+    if (typeof id !== 'number') {
+        throw new TypeError('insertDedupeCapEvent: INSERT returned no row id');
+    }
+    return id;
+}

package/lib/archive/db-ops/dedupe-cap/list-dedupe-cap-shape-keys.d.ts

@@ -0,0 +1,21 @@
+import type { Knex } from 'knex';
+/**
+ * Every distinct `dedupe_cap_events.shape_key` recorded in this archive —
+ * used by `CrawlerOrchestrator` to preload `DedupeCapTracker`'s sticky set
+ * on `--resume` / `--append` / `--retry-failed` / `--inventory`, so a trap
+ * this crawl already paid the cost of discovering once is not re-admitted
+ * in a later session. Fresh (non-resuming) crawls do not call this — there
+ * is no archive history to seed from.
+ *
+ * Unlike `listDnsBurnedHostCandidates`, no additional exclusion logic is
+ * needed: once `DedupeCapTracker` confirms a shape as a trap, it stays
+ * confirmed — there is no equivalent of "the host might have recovered
+ * since".
+ *
+ * Returns `[]` on legacy archives that pre-date the `dedupe_cap_events`
+ * table (self-healed on next writer open, so this is never a permanent
+ * state) or that have recorded no capped shapes.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns Distinct shape keys already confirmed capped.
+ */
+export declare function listDedupeCapShapeKeys(knex: Knex): Promise<string[]>;

package/lib/archive/db-ops/dedupe-cap/list-dedupe-cap-shape-keys.js

@@ -0,0 +1,27 @@
+/**
+ * Every distinct `dedupe_cap_events.shape_key` recorded in this archive —
+ * used by `CrawlerOrchestrator` to preload `DedupeCapTracker`'s sticky set
+ * on `--resume` / `--append` / `--retry-failed` / `--inventory`, so a trap
+ * this crawl already paid the cost of discovering once is not re-admitted
+ * in a later session. Fresh (non-resuming) crawls do not call this — there
+ * is no archive history to seed from.
+ *
+ * Unlike `listDnsBurnedHostCandidates`, no additional exclusion logic is
+ * needed: once `DedupeCapTracker` confirms a shape as a trap, it stays
+ * confirmed — there is no equivalent of "the host might have recovered
+ * since".
+ *
+ * Returns `[]` on legacy archives that pre-date the `dedupe_cap_events`
+ * table (self-healed on next writer open, so this is never a permanent
+ * state) or that have recorded no capped shapes.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns Distinct shape keys already confirmed capped.
+ */
+export async function listDedupeCapShapeKeys(knex) {
+    const hasTable = await knex.schema.hasTable('dedupe_cap_events');
+    if (!hasTable) {
+        return [];
+    }
+    const rows = (await knex('dedupe_cap_events').distinct('shape_key'));
+    return rows.map((row) => row.shape_key);
+}

package/lib/archive/types.d.ts

@@ -186,6 +186,19 @@ export interface InsertNetworkOutageParams {
     triggerErrorCount: number;
     triggerHostCount: number;
 }
+/**
+ * Fields required to record a newly-capped URL shape via
+ * `Database.insertDedupeCapEvent`. camelCase, mapped to snake_case columns
+ * on write — same convention as {@link InsertNetworkOutageParams}.
+ */
+export interface InsertDedupeCapEventParams {
+    shapeKey: string;
+    sampleUrl: string;
+    bodyHash: Buffer;
+    effectiveThreshold: number;
+    observedCount: number;
+    detectedAt: number;
+}
 /**
  * Filter type for querying pages from the database.
  *

package/lib/classify-error-kind.d.ts

@@ -14,6 +14,7 @@ import type { ErrorKind } from './types.js';
  * classifyErrorKind('getaddrinfo ENOTFOUND www.example.com'); // 'dns'
  * classifyErrorKind('gave up after 3 retries — Race 180,000ms'); // 'timeout'
  * classifyErrorKind('Protocol error (Page.reload): Target closed'); // 'protocol'
+ * classifyErrorKind('Maximum number of redirects exceeded'); // 'redirect-loop'
  * ```
  */
 export declare function classifyErrorKind(message: string): ErrorKind;

package/lib/classify-error-kind.js

@@ -53,6 +53,19 @@ const MATCHERS = [
         kind: 'parse-error',
         pattern: /Parse Error|Expected HTTP\/|Unexpected end of stream/i,
     },
+    // `follow-redirects` (pinned at 1.16.0, `fetch-destination.ts`'s HEAD/GET
+    // pre-flight) throws exactly "Maximum number of redirects exceeded" —
+    // no cause token, no URL — when a chain never terminates within its
+    // `maxRedirects` budget. `ERR_TOO_MANY_REDIRECTS` is the equivalent
+    // Chromium net-error code, covering the same symptom surfaced through a
+    // puppeteer navigation instead of the Node HTTP client. Both mean the
+    // SAME thing: the site's own redirect configuration loops and will loop
+    // again on any future fetch, which is why this is deterministic (not a
+    // transient network condition) — see `PERMANENT_ERROR_KINDS`.
+    {
+        kind: 'redirect-loop',
+        pattern: /Maximum number of redirects exceeded|ERR_TOO_MANY_REDIRECTS/i,
+    },
     // `client-blocked` covers Chromium's ERR_BLOCKED_* family — the browser
     // actively decided to reject the request (ad/tracker heuristics, CSP,
     // CORB / ORB, administrator block list, fingerprinting protection,
@@ -110,6 +123,7 @@ const MATCHERS = [
  * classifyErrorKind('getaddrinfo ENOTFOUND www.example.com'); // 'dns'
  * classifyErrorKind('gave up after 3 retries — Race 180,000ms'); // 'timeout'
  * classifyErrorKind('Protocol error (Page.reload): Target closed'); // 'protocol'
+ * classifyErrorKind('Maximum number of redirects exceeded'); // 'redirect-loop'
  * ```
  */
 export function classifyErrorKind(message) {

package/lib/crawler/assert-chrome-installed.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Verifies that Puppeteer can resolve an installed Chrome/Chromium executable
+ * before a crawl starts.
+ *
+ * A crawl otherwise only discovers a missing browser deep inside the
+ * per-URL scrape loop (`Crawler#_launchBrowserAndScrape`), where it surfaces
+ * as one more scrape error among many — the CLI still prints "Crawl
+ * completed" and writes an archive, so a missing Chrome (a fatal
+ * precondition, not a per-page failure) is easy to miss. Calling this once,
+ * before any archive I/O begins, turns it into an immediate, actionable
+ * failure instead.
+ * @param executablePath - Explicit override, matching
+ *   {@link CrawlerOptions.executablePath}. Pass `null` (or omit) to check
+ *   Puppeteer's own pinned Chrome resolution instead.
+ * @throws {Error} When the resolved executable path does not exist on disk.
+ * @example
+ * ```ts
+ * import { assertChromeIsInstalled } from '@nitpicker/crawler';
+ *
+ * // Throws with install instructions before any crawl work starts.
+ * await assertChromeIsInstalled();
+ * ```
+ */
+export declare function assertChromeIsInstalled(executablePath?: string | null): Promise<void>;