@nitpicker/crawler 0.15.0 → 0.17.0

package/lib/archive/archive.d.ts

@@ -1,5 +1,5 @@
 import type { TemplateClusterReason } from './db-ops/analysis/types.js';
-import type { Config, InsertNetworkOutageParams, InventoryRunMeta, PageSource } from './types.js';
+import type { Config, InsertDedupeCapEventParams, InsertNetworkOutageParams, InventoryRunMeta, PageSource } from './types.js';
 import type { OutageWindow } from '../is-within-outage-window.js';
 import type { PageData, CrawlerError, Resource } from '../utils/types/types.js';
 import type { ConsoleLogEntry } from '@d-zero/beholder';
@@ -49,6 +49,17 @@ export default class Archive extends ArchiveAccessor {
      * @deprecated This method is no longer functional.
      */
     abort(): void;
+    /**
+     * Adds onto the `rejected_count` of a shape's `dedupe_cap_events` row,
+     * looked up by `shape_key` rather than `id` — used for a shape that
+     * capped in an earlier session (preloaded into `DedupeCapTracker`'s
+     * sticky set) and so has no event id from the current session.
+     *
+     * Thin facade over {@link Database.accumulateDedupeCapRejectedCount}.
+     * @param shapeKey - The capped shape whose rejection count to accumulate.
+     * @param rejectedCount - Additional anchors rejected for this shape in the current session.
+     */
+    accumulateDedupeCapRejectedCount(shapeKey: string, rejectedCount: number): Promise<void>;
     /**
      * Records a crawler-level error to both the human-readable `error.log` (full
      * stack, for debugging) and the structured `crawl_errors` table (queryable,
@@ -80,6 +91,15 @@ export default class Archive extends ArchiveAccessor {
      * @param endedAt - Epoch ms the outage is considered to have ended.
      */
     closeNetworkOutage(id: number, endedAt: number): Promise<void>;
+    /**
+     * Finalizes a `dedupe_cap_events` row by stamping `rejected_count` — a
+     * no-op if already finalized.
+     *
+     * Thin facade over {@link Database.finalizeDedupeCapEvent}.
+     * @param id - The `dedupe_cap_events.id` to finalize.
+     * @param rejectedCount - Number of anchors rejected for this shape after it capped.
+     */
+    finalizeDedupeCapEvent(id: number, rejectedCount: number): Promise<void>;
     /**
      * Retrieves the current crawling state, including lists of scraped and pending URLs.
      * @returns An object with `scraped` and `pending` URL arrays.
@@ -136,6 +156,15 @@ export default class Archive extends ArchiveAccessor {
      * @returns The base URL string.
      */
     getUrl(): Promise<string>;
+    /**
+     * Appends one row (`rejected_count = NULL`) to the `dedupe_cap_events`
+     * journal.
+     *
+     * Thin facade over {@link Database.insertDedupeCapEvent}.
+     * @param params - The newly-capped shape's fields to record.
+     * @returns The autoincremented `id` of the inserted row.
+     */
+    insertDedupeCapEvent(params: InsertDedupeCapEventParams): Promise<number>;
     /**
      * Pre-insert inventory non-HTML URLs as `source='inventory-seed'`
      * placeholders in the `resources` table — the non-HTML counterpart of
@@ -165,6 +194,19 @@ export default class Archive extends ArchiveAccessor {
      * @param urls - HTML seed URLs to pre-insert. No-op when empty.
      */
     insertInventorySeeds(urls: readonly ExURL[]): Promise<void>;
+    /**
+     * Records exclude-matched inventory URLs as terminal skipped pages —
+     * the same `is_skipped=1, skip_reason='excluded'` state the normal
+     * crawl's fetch-time gate writes for link-discovered excluded URLs,
+     * labelled `source='inventory-seed'`. Thin facade over
+     * {@link Database.insertInventorySkippedPages} — see the underlying
+     * op's JSDoc for the parity rationale and crawled-wins safety.
+     *
+     * `ExURL` inputs are normalised to `withoutHashAndAuth` here for the
+     * same storage-key consistency reason as {@link insertInventorySeeds}.
+     * @param urls - Exclude-matched URLs to record. No-op when empty.
+     */
+    insertInventorySkippedPages(urls: readonly ExURL[]): Promise<void>;
     /**
      * Appends one open row to the `network_outages` journal.
      *
@@ -174,6 +216,15 @@ export default class Archive extends ArchiveAccessor {
      * @returns The autoincremented `id` of the inserted row.
      */
     insertNetworkOutage(params: InsertNetworkOutageParams): Promise<number>;
+    /**
+     * Every distinct `dedupe_cap_events.shape_key` recorded in this archive.
+     * Consumed by `CrawlerOrchestrator` to preload `DedupeCapTracker`'s
+     * sticky set on `--resume` / `--append` / `--retry-failed` /
+     * `--inventory`, mirroring {@link listDnsBurnedHostCandidates}'s
+     * writer-only exposure.
+     * @returns Distinct shape keys already confirmed capped.
+     */
+    listDedupeCapShapeKeys(): Promise<string[]>;
     /**
      * Hostnames whose `crawl_errors` history is consistently DNS failures and
      * for which no recent 2xx/3xx page or resource is recorded. Consumed by

package/lib/archive/archive.js

@@ -94,6 +94,20 @@ export default class Archive extends ArchiveAccessor {
      * @deprecated This method is no longer functional.
      */
     abort() { }
+    /**
+     * Adds onto the `rejected_count` of a shape's `dedupe_cap_events` row,
+     * looked up by `shape_key` rather than `id` — used for a shape that
+     * capped in an earlier session (preloaded into `DedupeCapTracker`'s
+     * sticky set) and so has no event id from the current session.
+     *
+     * Thin facade over {@link Database.accumulateDedupeCapRejectedCount}.
+     * @param shapeKey - The capped shape whose rejection count to accumulate.
+     * @param rejectedCount - Additional anchors rejected for this shape in the current session.
+     */
+    async accumulateDedupeCapRejectedCount(shapeKey, rejectedCount) {
+        dbLog('Accumulate dedupe cap rejected count shapeKey=%s rejectedCount=%d', shapeKey, rejectedCount);
+        return await this.#db.accumulateDedupeCapRejectedCount(shapeKey, rejectedCount);
+    }
     /**
      * Records a crawler-level error to both the human-readable `error.log` (full
      * stack, for debugging) and the structured `crawl_errors` table (queryable,
@@ -135,6 +149,18 @@ export default class Archive extends ArchiveAccessor {
         dbLog('Close network outage id=%d endedAt=%d', id, endedAt);
         return await this.#db.closeNetworkOutage(id, endedAt);
     }
+    /**
+     * Finalizes a `dedupe_cap_events` row by stamping `rejected_count` — a
+     * no-op if already finalized.
+     *
+     * Thin facade over {@link Database.finalizeDedupeCapEvent}.
+     * @param id - The `dedupe_cap_events.id` to finalize.
+     * @param rejectedCount - Number of anchors rejected for this shape after it capped.
+     */
+    async finalizeDedupeCapEvent(id, rejectedCount) {
+        dbLog('Finalize dedupe cap event id=%d rejectedCount=%d', id, rejectedCount);
+        return await this.#db.finalizeDedupeCapEvent(id, rejectedCount);
+    }
     /**
      * Retrieves the current crawling state, including lists of scraped and pending URLs.
      * @returns An object with `scraped` and `pending` URL arrays.
@@ -202,6 +228,18 @@ export default class Archive extends ArchiveAccessor {
     async getUrl() {
         return this.#db.getBaseUrl();
     }
+    /**
+     * Appends one row (`rejected_count = NULL`) to the `dedupe_cap_events`
+     * journal.
+     *
+     * Thin facade over {@link Database.insertDedupeCapEvent}.
+     * @param params - The newly-capped shape's fields to record.
+     * @returns The autoincremented `id` of the inserted row.
+     */
+    async insertDedupeCapEvent(params) {
+        dbLog('Insert dedupe cap event: shapeKey=%s', params.shapeKey);
+        return await this.#db.insertDedupeCapEvent(params);
+    }
     /**
      * Pre-insert inventory non-HTML URLs as `source='inventory-seed'`
      * placeholders in the `resources` table — the non-HTML counterpart of
@@ -243,6 +281,25 @@ export default class Archive extends ArchiveAccessor {
         dbLog('Insert inventory seeds: %d URL(s)', urls.length);
         await this.#db.insertInventorySeeds(urls.map((u) => u.withoutHashAndAuth));
     }
+    /**
+     * Records exclude-matched inventory URLs as terminal skipped pages —
+     * the same `is_skipped=1, skip_reason='excluded'` state the normal
+     * crawl's fetch-time gate writes for link-discovered excluded URLs,
+     * labelled `source='inventory-seed'`. Thin facade over
+     * {@link Database.insertInventorySkippedPages} — see the underlying
+     * op's JSDoc for the parity rationale and crawled-wins safety.
+     *
+     * `ExURL` inputs are normalised to `withoutHashAndAuth` here for the
+     * same storage-key consistency reason as {@link insertInventorySeeds}.
+     * @param urls - Exclude-matched URLs to record. No-op when empty.
+     */
+    async insertInventorySkippedPages(urls) {
+        if (urls.length === 0) {
+            return;
+        }
+        dbLog('Insert inventory skipped pages: %d URL(s)', urls.length);
+        await this.#db.insertInventorySkippedPages(urls.map((u) => u.withoutHashAndAuth));
+    }
     /**
      * Appends one open row to the `network_outages` journal.
      *
@@ -255,6 +312,17 @@ export default class Archive extends ArchiveAccessor {
         dbLog('Insert network outage: startedAt=%d probeHost=%s', params.startedAt, params.probeHost);
         return await this.#db.insertNetworkOutage(params);
     }
+    /**
+     * Every distinct `dedupe_cap_events.shape_key` recorded in this archive.
+     * Consumed by `CrawlerOrchestrator` to preload `DedupeCapTracker`'s
+     * sticky set on `--resume` / `--append` / `--retry-failed` /
+     * `--inventory`, mirroring {@link listDnsBurnedHostCandidates}'s
+     * writer-only exposure.
+     * @returns Distinct shape keys already confirmed capped.
+     */
+    async listDedupeCapShapeKeys() {
+        return this.#db.listDedupeCapShapeKeys();
+    }
     /**
      * Hostnames whose `crawl_errors` history is consistently DNS failures and
      * for which no recent 2xx/3xx page or resource is recorded. Consumed by

package/lib/archive/create-adjunct-tables.d.ts

@@ -16,6 +16,9 @@ import type { Knex } from 'knex';
  * - `inventory_runs` — `--inventory` audit log (no FK; append-only)
  * - `network_outages` — operator-network-outage journal (no FK; append-only
  *   except `ended_at`, which is written once on recovery)
+ * - `dedupe_cap_events` — `--dedupe-cap` same-cluster soft-cap audit log (no
+ *   FK; append-only except `rejected_count`, which is written once at
+ *   `crawlEnd`)
  * - `analysis_text_refs` + `analysis_violations` — analyze-phase findings,
  *   FK → `content_items(id)`
  * - `page_templates` — DOM-structure template classification (`--templates`,

package/lib/archive/create-adjunct-tables.js

@@ -15,6 +15,9 @@
  * - `inventory_runs` — `--inventory` audit log (no FK; append-only)
  * - `network_outages` — operator-network-outage journal (no FK; append-only
  *   except `ended_at`, which is written once on recovery)
+ * - `dedupe_cap_events` — `--dedupe-cap` same-cluster soft-cap audit log (no
+ *   FK; append-only except `rejected_count`, which is written once at
+ *   `crawlEnd`)
  * - `analysis_text_refs` + `analysis_violations` — analyze-phase findings,
  *   FK → `content_items(id)`
  * - `page_templates` — DOM-structure template classification (`--templates`,
@@ -290,6 +293,7 @@ export async function createAdjunctTables(instance) {
             t.integer('new_pages').nullable();
             t.integer('new_resources').nullable();
             t.integer('scope_skipped').nullable();
+            t.integer('exclude_skipped').nullable();
             t.integer('invalid_skipped').nullable();
             t.text('notes').nullable();
             t.index('ran_at');
@@ -326,6 +330,45 @@ export async function createAdjunctTables(instance) {
             t.integer('trigger_host_count').notNullable();
         });
     }
+    if (!(await instance.schema.hasTable('dedupe_cap_events'))) {
+        await instance.schema.createTable('dedupe_cap_events', (t) => {
+            // One row per URL shape the `--dedupe-cap` same-cluster soft cap
+            // (`DedupeCapTracker`) confirmed as a trap during this crawl. No
+            // index: a crawl produces at most a handful of these rows (same
+            // reasoning as `network_outages`, above).
+            t.increments('id');
+            // The URL shape key (`computeShapeKey`) that capped — a template
+            // with placeholders (e.g. `example.com/news/date/{n}/`), not a
+            // literal URL.
+            t.string('shape_key').notNullable();
+            // One concrete URL matching this shape, captured at cap time so a
+            // human reading the audit log can identify what was being
+            // crawled — `shape_key` alone is a template, not a navigable URL.
+            t.string('sample_url').notNullable();
+            // `computeBodyHash` result recorded at cap time. Nullable only in
+            // the sense that BLOB columns are nullable by default; every row
+            // this feature writes populates it (a page with no rendered
+            // `<body>` never reaches the tracker — see `Crawler#handleResult`).
+            t.binary('body_hash').nullable();
+            // The Misra-Gries threshold that actually triggered the cap,
+            // after halving for the `body_hash`-match / `og:url`-mismatch
+            // confidence signals — NOT necessarily equal to `--dedupe-cap`'s
+            // raw value.
+            t.integer('effective_threshold').notNullable();
+            // The tracker's Misra-Gries counter value at cap time: a LOWER
+            // BOUND on the number of matching-signature pages seen for this
+            // shape, not an exact observation count (see `DedupeCapTracker`).
+            t.integer('observed_count').notNullable();
+            t.integer('detected_at').notNullable();
+            // NULL until `crawlEnd` finalizes it (see
+            // `Crawler#getDedupeCapRejections`). Unlike `network_outages.ended_at`,
+            // a NULL here has no ambiguous "still ongoing" reading — a
+            // crawl that never reached `crawlEnd` simply left the count
+            // undetermined, so no boot-time reconciliation pass is needed
+            // (readers display "unknown", not "0" or "unbounded").
+            t.integer('rejected_count').nullable();
+        });
+    }
     if (!(await instance.schema.hasTable('analysis_text_refs'))) {
         await instance.raw(`
 			CREATE TABLE analysis_text_refs (

package/lib/archive/create-entity-tables.js

@@ -198,6 +198,7 @@ export async function createEntityTables(instance) {
 			header_set_id    INTEGER REFERENCES header_sets(id),
 			redirect_dest_id INTEGER REFERENCES content_items(id) DEFERRABLE INITIALLY DEFERRED,
 			alias_of_id      INTEGER REFERENCES content_items(id) DEFERRABLE INITIALLY DEFERRED,
+			dedupe_cap_event_id INTEGER REFERENCES dedupe_cap_events(id) DEFERRABLE INITIALLY DEFERRED,
 			source           TEXT NOT NULL DEFAULT 'crawled',
 			first_crawled_at INTEGER,
 			last_crawled_at  INTEGER,
@@ -218,6 +219,15 @@ export async function createEntityTables(instance) {
     // `migrateContentItemsAliasOfId` instead, which runs after the
     // column-add guard for both fresh and legacy archives (same reasoning as
     // `page_meta.body_hash`'s index).
+    //
+    // `dedupe_cap_event_id` gets no index anywhere, not even in its own
+    // migration (`migrateContentItemsDedupeCapEventId`) — unlike
+    // `alias_of_id`, there is no known hot read path filtering on this
+    // column yet (`--dedupe-cap` is opt-in and the marked row count is
+    // small: capped shapes × matching URLs). Adding a speculative index
+    // without a measured query to justify it violates this archive's
+    // "no speculative index" rule; add one later with `EXPLAIN QUERY PLAN`
+    // evidence if a real hot path emerges.
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_content_type_id ON content_items(content_type_id)');
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_crawl_order ON content_items(crawl_order)');
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_source ON content_items(source)');

package/lib/archive/database.d.ts

@@ -1,6 +1,6 @@
 import type { TemplateClusterReason } from './db-ops/analysis/types.js';
 import type { JsonLdRow, MainContentAudioRow, MainContentButtonRow, MainContentCanvasRow, MainContentHeadingRow, MainContentIframeRow, MainContentImageRow, MainContentTableRow, MainContentVideoRow, TagRow } from './meta/types.js';
-import type { Config, DatabaseOption, DB_Redirect, DB_Resource, DatabaseEvent, InsertNetworkOutageParams, InventoryRunMeta, PageFilter, PageSource } from './types.js';
+import type { Config, DatabaseOption, DB_Redirect, DB_Resource, DatabaseEvent, InsertDedupeCapEventParams, InsertNetworkOutageParams, InventoryRunMeta, PageFilter, PageSource } from './types.js';
 import type { OutageWindow } from '../is-within-outage-window.js';
 import type { PageData, Resource } from '../utils/types/types.js';
 import type { ConsoleLogEntry } from '@d-zero/beholder';
@@ -32,6 +32,14 @@ import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-awa
 export declare class Database extends EventEmitter<DatabaseEvent> {
     #private;
     private constructor();
+    /**
+     * Adds onto the `rejected_count` of a shape's `dedupe_cap_events` row,
+     * looked up by `shape_key` rather than `id`. Delegates to
+     * {@link accumulateDedupeCapRejectedCountOp}.
+     * @param shapeKey - The capped shape whose rejection count to accumulate.
+     * @param rejectedCount - Additional anchors rejected for this shape in the current session.
+     */
+    accumulateDedupeCapRejectedCount(shapeKey: string, rejectedCount: number): Promise<void>;
     /**
      * Forces a WAL checkpoint, writing all pending WAL data back to the main
      * database file. Delegates to {@link checkpointOp}.
@@ -49,6 +57,14 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * Delegates to {@link destroyOp}.
      */
     destroy(): Promise<void>;
+    /**
+     * Finalizes a `dedupe_cap_events` row by stamping `rejected_count` — a
+     * no-op if the row is already finalized. Delegates to
+     * {@link finalizeDedupeCapEventOp}.
+     * @param id - The `dedupe_cap_events.id` to finalize.
+     * @param rejectedCount - Number of anchors rejected for this shape after it capped.
+     */
+    finalizeDedupeCapEvent(id: number, rejectedCount: number): Promise<void>;
     /**
      * Retrieves all anchors (outgoing links) on a specific page.
      * Delegates to {@link getAnchorsOnPageOp}.
@@ -273,6 +289,13 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @param isExternal - Whether the URL is external to the crawl scope.
      */
     insertCrawlError(url: string | null, message: string, isExternal?: boolean): Promise<void>;
+    /**
+     * Appends one row (`rejected_count = NULL`) to the `dedupe_cap_events`
+     * journal. Delegates to {@link insertDedupeCapEventOp}.
+     * @param params - The newly-capped shape's fields to record.
+     * @returns The autoincremented `id` of the newly-inserted row.
+     */
+    insertDedupeCapEvent(params: InsertDedupeCapEventParams): Promise<number>;
     /**
      * Pre-insert inventory non-HTML URLs into `resources` as placeholder rows.
      * Delegates to {@link insertInventoryResourcesOp}.
@@ -287,6 +310,15 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @param urls - URL strings already in `withoutHashAndAuth` form.
      */
     insertInventorySeeds(urls: readonly string[]): Promise<void>;
+    /**
+     * Records exclude-matched inventory URLs as terminal skipped pages
+     * (`scraped=1`, `is_skipped=1`, `skip_reason='excluded'`,
+     * `source='inventory-seed'`). Delegates to
+     * {@link insertInventorySkippedPagesOp} — see that op's JSDoc for the
+     * normal-crawl parity rationale.
+     * @param urls - URL strings already in `withoutHashAndAuth` form.
+     */
+    insertInventorySkippedPages(urls: readonly string[]): Promise<void>;
     /**
      * Appends one open (`ended_at = NULL`) row to the `network_outages`
      * journal. Delegates to {@link insertNetworkOutageOp}.
@@ -317,6 +349,13 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @param pageUrl - The URL of the page that references the resource.
      */
     insertResourceReferrers(src: string, pageUrl: string): Promise<void>;
+    /**
+     * Every distinct `dedupe_cap_events.shape_key` recorded in this archive.
+     * Delegates to {@link listDedupeCapShapeKeysOp}.
+     * @returns Distinct shape keys already confirmed capped, or `[]` on an
+     *   archive that predates `dedupe_cap_events` or has recorded none.
+     */
+    listDedupeCapShapeKeys(): Promise<string[]>;
     /**
      * Hostnames whose `crawl_errors` history is consistently DNS failures and
      * for which no recent 2xx-3xx page or resource is recorded.

package/lib/archive/database.js

@@ -16,6 +16,10 @@ import { getName as getNameOp } from './db-ops/config/get-name.js';
 import { setConfig as setConfigOp } from './db-ops/config/set-config.js';
 import { updateConfig as updateConfigOp } from './db-ops/config/update-config.js';
 import { replaceConsoleLogs as replaceConsoleLogsOp } from './db-ops/console-logs/replace-console-logs.js';
+import { accumulateDedupeCapRejectedCount as accumulateDedupeCapRejectedCountOp } from './db-ops/dedupe-cap/accumulate-dedupe-cap-rejected-count.js';
+import { finalizeDedupeCapEvent as finalizeDedupeCapEventOp } from './db-ops/dedupe-cap/finalize-dedupe-cap-event.js';
+import { insertDedupeCapEvent as insertDedupeCapEventOp } from './db-ops/dedupe-cap/insert-dedupe-cap-event.js';
+import { listDedupeCapShapeKeys as listDedupeCapShapeKeysOp } from './db-ops/dedupe-cap/list-dedupe-cap-shape-keys.js';
 import { insertCrawlError as insertCrawlErrorOp } from './db-ops/errors/insert-crawl-error.js';
 import { insertPageError as insertPageErrorOp } from './db-ops/errors/insert-page-error.js';
 import { listDnsBurnedHostCandidates as listDnsBurnedHostCandidatesOp } from './db-ops/errors/list-dns-burned-host-candidates.js';
@@ -48,6 +52,7 @@ import { getScrapedHtmlPageCount as getScrapedHtmlPageCountOp } from './db-ops/p
 import { repromoteExternalPages as repromoteExternalPagesOp } from './db-ops/pages/reset/repromote-external-pages.js';
 import { resetFailedPages as resetFailedPagesOp } from './db-ops/pages/reset/reset-failed-pages.js';
 import { insertInventorySeeds as insertInventorySeedsOp } from './db-ops/pages/write/insert-inventory-seeds.js';
+import { insertInventorySkippedPages as insertInventorySkippedPagesOp } from './db-ops/pages/write/insert-inventory-skipped-pages.js';
 import { recordRedirect as recordRedirectOp } from './db-ops/pages/write/record-redirect.js';
 import { setSkippedPage as setSkippedPageOp } from './db-ops/pages/write/set-skipped-page.js';
 import { updatePage as updatePageOp } from './db-ops/pages/write/update-page.js';
@@ -118,6 +123,16 @@ export class Database extends EventEmitter {
             },
         });
     }
+    /**
+     * Adds onto the `rejected_count` of a shape's `dedupe_cap_events` row,
+     * looked up by `shape_key` rather than `id`. Delegates to
+     * {@link accumulateDedupeCapRejectedCountOp}.
+     * @param shapeKey - The capped shape whose rejection count to accumulate.
+     * @param rejectedCount - Additional anchors rejected for this shape in the current session.
+     */
+    async accumulateDedupeCapRejectedCount(shapeKey, rejectedCount) {
+        return emitErrorAndRetry(this, 'Database.accumulateDedupeCapRejectedCount', async () => await accumulateDedupeCapRejectedCountOp(this.#instance, shapeKey, rejectedCount), retrySetting);
+    }
     /**
      * Forces a WAL checkpoint, writing all pending WAL data back to the main
      * database file. Delegates to {@link checkpointOp}.
@@ -141,6 +156,16 @@ export class Database extends EventEmitter {
     async destroy() {
         await destroyOp(this.#instance);
     }
+    /**
+     * Finalizes a `dedupe_cap_events` row by stamping `rejected_count` — a
+     * no-op if the row is already finalized. Delegates to
+     * {@link finalizeDedupeCapEventOp}.
+     * @param id - The `dedupe_cap_events.id` to finalize.
+     * @param rejectedCount - Number of anchors rejected for this shape after it capped.
+     */
+    async finalizeDedupeCapEvent(id, rejectedCount) {
+        return emitErrorAndRetry(this, 'Database.finalizeDedupeCapEvent', async () => await finalizeDedupeCapEventOp(this.#instance, id, rejectedCount), retrySetting);
+    }
     /**
      * Retrieves all anchors (outgoing links) on a specific page.
      * Delegates to {@link getAnchorsOnPageOp}.
@@ -422,6 +447,15 @@ export class Database extends EventEmitter {
     async insertCrawlError(url, message, isExternal = false) {
         return emitErrorAndRetry(this, 'Database.insertCrawlError', async () => await insertCrawlErrorOp(this.#instance, url, message, isExternal), retrySetting);
     }
+    /**
+     * Appends one row (`rejected_count = NULL`) to the `dedupe_cap_events`
+     * journal. Delegates to {@link insertDedupeCapEventOp}.
+     * @param params - The newly-capped shape's fields to record.
+     * @returns The autoincremented `id` of the newly-inserted row.
+     */
+    async insertDedupeCapEvent(params) {
+        return emitErrorAndRetry(this, 'Database.insertDedupeCapEvent', async () => await insertDedupeCapEventOp(this.#instance, params), retrySetting);
+    }
     /**
      * Pre-insert inventory non-HTML URLs into `resources` as placeholder rows.
      * Delegates to {@link insertInventoryResourcesOp}.
@@ -440,6 +474,17 @@ export class Database extends EventEmitter {
     async insertInventorySeeds(urls) {
         return emitErrorAndRetry(this, 'Database.insertInventorySeeds', async () => await insertInventorySeedsOp(this.#instance, this.#writeRefCaches, urls), retrySetting);
     }
+    /**
+     * Records exclude-matched inventory URLs as terminal skipped pages
+     * (`scraped=1`, `is_skipped=1`, `skip_reason='excluded'`,
+     * `source='inventory-seed'`). Delegates to
+     * {@link insertInventorySkippedPagesOp} — see that op's JSDoc for the
+     * normal-crawl parity rationale.
+     * @param urls - URL strings already in `withoutHashAndAuth` form.
+     */
+    async insertInventorySkippedPages(urls) {
+        return emitErrorAndRetry(this, 'Database.insertInventorySkippedPages', async () => await insertInventorySkippedPagesOp(this.#instance, this.#writeRefCaches, urls), retrySetting);
+    }
     /**
      * Appends one open (`ended_at = NULL`) row to the `network_outages`
      * journal. Delegates to {@link insertNetworkOutageOp}.
@@ -478,6 +523,15 @@ export class Database extends EventEmitter {
     async insertResourceReferrers(src, pageUrl) {
         return emitErrorAndRetry(this, 'Database.insertResourceReferrers', async () => await insertResourceReferrersOp(this.#instance, this.#writeRefCaches, src, pageUrl), retrySetting);
     }
+    /**
+     * Every distinct `dedupe_cap_events.shape_key` recorded in this archive.
+     * Delegates to {@link listDedupeCapShapeKeysOp}.
+     * @returns Distinct shape keys already confirmed capped, or `[]` on an
+     *   archive that predates `dedupe_cap_events` or has recorded none.
+     */
+    async listDedupeCapShapeKeys() {
+        return emitErrorAndRetry(this, 'Database.listDedupeCapShapeKeys', async () => await listDedupeCapShapeKeysOp(this.#instance), retrySetting);
+    }
     /**
      * Hostnames whose `crawl_errors` history is consistently DNS failures and
      * for which no recent 2xx-3xx page or resource is recorded.

package/lib/archive/db-ops/config/get-config.js

@@ -19,6 +19,7 @@ export async function getConfig(knex) {
         excludeUrls: getJSON(config.excludeUrls, []),
         roots: getJSON(config.roots, []),
         retry: config.retry ?? 3,
+        maxExcludedDepth: config.maxExcludedDepth ?? 0,
     };
     // @ts-expect-error — `id` is the primary key, not part of the public Config shape
     delete opt.id;

package/lib/archive/db-ops/dedupe-cap/accumulate-dedupe-cap-rejected-count.d.ts

@@ -0,0 +1,18 @@
+import type { Knex } from 'knex';
+/**
+ * Adds `rejectedCount` onto the `rejected_count` of the `dedupe_cap_events`
+ * row for `shapeKey`, treating a still-`NULL` count as `0`. Unlike
+ * `finalize-dedupe-cap-event.ts` (which stamps a session's own newly-capped
+ * shape exactly once, guarded by `whereNull`), this targets a shape that
+ * capped in an EARLIER session and was preloaded into `DedupeCapTracker`'s
+ * sticky set (see `DedupeCapTracker`'s constructor JSDoc) — gate rejections
+ * for such a shape still occur in the current session, but no `dedupeCap`
+ * event (and thus no new row) is ever emitted for it, since the tracker
+ * short-circuits on an already-sticky shape before `observe` runs. Matches
+ * by `shape_key` rather than `id` because the caller (`CrawlerOrchestrator`)
+ * only has the shape key for a preloaded-sticky shape, never its row id.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param shapeKey - The capped shape whose rejection count to accumulate.
+ * @param rejectedCount - Additional anchors rejected for this shape in the current session.
+ */
+export declare function accumulateDedupeCapRejectedCount(knex: Knex, shapeKey: string, rejectedCount: number): Promise<void>;

package/lib/archive/db-ops/dedupe-cap/accumulate-dedupe-cap-rejected-count.js

@@ -0,0 +1,23 @@
+/**
+ * Adds `rejectedCount` onto the `rejected_count` of the `dedupe_cap_events`
+ * row for `shapeKey`, treating a still-`NULL` count as `0`. Unlike
+ * `finalize-dedupe-cap-event.ts` (which stamps a session's own newly-capped
+ * shape exactly once, guarded by `whereNull`), this targets a shape that
+ * capped in an EARLIER session and was preloaded into `DedupeCapTracker`'s
+ * sticky set (see `DedupeCapTracker`'s constructor JSDoc) — gate rejections
+ * for such a shape still occur in the current session, but no `dedupeCap`
+ * event (and thus no new row) is ever emitted for it, since the tracker
+ * short-circuits on an already-sticky shape before `observe` runs. Matches
+ * by `shape_key` rather than `id` because the caller (`CrawlerOrchestrator`)
+ * only has the shape key for a preloaded-sticky shape, never its row id.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param shapeKey - The capped shape whose rejection count to accumulate.
+ * @param rejectedCount - Additional anchors rejected for this shape in the current session.
+ */
+export async function accumulateDedupeCapRejectedCount(knex, shapeKey, rejectedCount) {
+    await knex('dedupe_cap_events')
+        .where({ shape_key: shapeKey })
+        .update({
+        rejected_count: knex.raw('COALESCE(rejected_count, 0) + ?', [rejectedCount]),
+    });
+}

package/lib/archive/db-ops/dedupe-cap/finalize-dedupe-cap-event.d.ts

@@ -0,0 +1,12 @@
+import type { Knex } from 'knex';
+/**
+ * Finalizes a `dedupe_cap_events` row by stamping `rejected_count` — but
+ * ONLY if it is still unset. The `whereNull('rejected_count')` guard makes
+ * this idempotent, mirroring `close-network-outage.ts`'s `ended_at` guard: a
+ * second call matches zero rows and is a silent no-op rather than
+ * overwriting an already-finalized count.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param id - The `dedupe_cap_events.id` to finalize.
+ * @param rejectedCount - Number of anchors rejected for this shape after it capped.
+ */
+export declare function finalizeDedupeCapEvent(knex: Knex, id: number, rejectedCount: number): Promise<void>;

package/lib/archive/db-ops/dedupe-cap/finalize-dedupe-cap-event.js

@@ -0,0 +1,15 @@
+/**
+ * Finalizes a `dedupe_cap_events` row by stamping `rejected_count` — but
+ * ONLY if it is still unset. The `whereNull('rejected_count')` guard makes
+ * this idempotent, mirroring `close-network-outage.ts`'s `ended_at` guard: a
+ * second call matches zero rows and is a silent no-op rather than
+ * overwriting an already-finalized count.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param id - The `dedupe_cap_events.id` to finalize.
+ * @param rejectedCount - Number of anchors rejected for this shape after it capped.
+ */
+export async function finalizeDedupeCapEvent(knex, id, rejectedCount) {
+    await knex('dedupe_cap_events').where({ id }).whereNull('rejected_count').update({
+        rejected_count: rejectedCount,
+    });
+}

package/lib/archive/db-ops/dedupe-cap/insert-dedupe-cap-event.d.ts

@@ -0,0 +1,14 @@
+import type { InsertDedupeCapEventParams } from '../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Appends one row to the `dedupe_cap_events` journal, with `rejected_count`
+ * left `NULL` — the row starts life without a finalized rejection count.
+ *
+ * Called the instant `DedupeCapTracker#observe` confirms a URL shape as a
+ * same-cluster trap (the `dedupeCap` event). See
+ * `finalize-dedupe-cap-event.ts` for how `rejected_count` is later set.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param params - The newly-capped shape's fields to record.
+ * @returns The autoincremented `id` of the newly-inserted row.
+ */
+export declare function insertDedupeCapEvent(knex: Knex, params: InsertDedupeCapEventParams): Promise<number>;

package/lib/archive/db-ops/dedupe-cap/insert-dedupe-cap-event.js

@@ -0,0 +1,30 @@
+/**
+ * Appends one row to the `dedupe_cap_events` journal, with `rejected_count`
+ * left `NULL` — the row starts life without a finalized rejection count.
+ *
+ * Called the instant `DedupeCapTracker#observe` confirms a URL shape as a
+ * same-cluster trap (the `dedupeCap` event). See
+ * `finalize-dedupe-cap-event.ts` for how `rejected_count` is later set.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param params - The newly-capped shape's fields to record.
+ * @returns The autoincremented `id` of the newly-inserted row.
+ */
+export async function insertDedupeCapEvent(knex, params) {
+    const inserted = await knex
+        .from('dedupe_cap_events')
+        .insert({
+        shape_key: params.shapeKey,
+        sample_url: params.sampleUrl,
+        body_hash: params.bodyHash,
+        effective_threshold: params.effectiveThreshold,
+        observed_count: params.observedCount,
+        detected_at: params.detectedAt,
+        rejected_count: null,
+    })
+        .returning('id');
+    const id = inserted[0]?.id;
+    if (typeof id !== 'number') {
+        throw new TypeError('insertDedupeCapEvent: INSERT returned no row id');
+    }
+    return id;
+}

package/lib/archive/db-ops/dedupe-cap/list-dedupe-cap-shape-keys.d.ts

@@ -0,0 +1,21 @@
+import type { Knex } from 'knex';
+/**
+ * Every distinct `dedupe_cap_events.shape_key` recorded in this archive —
+ * used by `CrawlerOrchestrator` to preload `DedupeCapTracker`'s sticky set
+ * on `--resume` / `--append` / `--retry-failed` / `--inventory`, so a trap
+ * this crawl already paid the cost of discovering once is not re-admitted
+ * in a later session. Fresh (non-resuming) crawls do not call this — there
+ * is no archive history to seed from.
+ *
+ * Unlike `listDnsBurnedHostCandidates`, no additional exclusion logic is
+ * needed: once `DedupeCapTracker` confirms a shape as a trap, it stays
+ * confirmed — there is no equivalent of "the host might have recovered
+ * since".
+ *
+ * Returns `[]` on legacy archives that pre-date the `dedupe_cap_events`
+ * table (self-healed on next writer open, so this is never a permanent
+ * state) or that have recorded no capped shapes.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns Distinct shape keys already confirmed capped.
+ */
+export declare function listDedupeCapShapeKeys(knex: Knex): Promise<string[]>;