@nitpicker/crawler 0.16.0 → 0.18.0

package/lib/archive/archive.d.ts

@@ -194,6 +194,19 @@ export default class Archive extends ArchiveAccessor {
      * @param urls - HTML seed URLs to pre-insert. No-op when empty.
      */
     insertInventorySeeds(urls: readonly ExURL[]): Promise<void>;
+    /**
+     * Records exclude-matched inventory URLs as terminal skipped pages —
+     * the same `is_skipped=1, skip_reason='excluded'` state the normal
+     * crawl's fetch-time gate writes for link-discovered excluded URLs,
+     * labelled `source='inventory-seed'`. Thin facade over
+     * {@link Database.insertInventorySkippedPages} — see the underlying
+     * op's JSDoc for the parity rationale and crawled-wins safety.
+     *
+     * `ExURL` inputs are normalised to `withoutHashAndAuth` here for the
+     * same storage-key consistency reason as {@link insertInventorySeeds}.
+     * @param urls - Exclude-matched URLs to record. No-op when empty.
+     */
+    insertInventorySkippedPages(urls: readonly ExURL[]): Promise<void>;
     /**
      * Appends one open row to the `network_outages` journal.
      *

package/lib/archive/archive.js

@@ -281,6 +281,25 @@ export default class Archive extends ArchiveAccessor {
         dbLog('Insert inventory seeds: %d URL(s)', urls.length);
         await this.#db.insertInventorySeeds(urls.map((u) => u.withoutHashAndAuth));
     }
+    /**
+     * Records exclude-matched inventory URLs as terminal skipped pages —
+     * the same `is_skipped=1, skip_reason='excluded'` state the normal
+     * crawl's fetch-time gate writes for link-discovered excluded URLs,
+     * labelled `source='inventory-seed'`. Thin facade over
+     * {@link Database.insertInventorySkippedPages} — see the underlying
+     * op's JSDoc for the parity rationale and crawled-wins safety.
+     *
+     * `ExURL` inputs are normalised to `withoutHashAndAuth` here for the
+     * same storage-key consistency reason as {@link insertInventorySeeds}.
+     * @param urls - Exclude-matched URLs to record. No-op when empty.
+     */
+    async insertInventorySkippedPages(urls) {
+        if (urls.length === 0) {
+            return;
+        }
+        dbLog('Insert inventory skipped pages: %d URL(s)', urls.length);
+        await this.#db.insertInventorySkippedPages(urls.map((u) => u.withoutHashAndAuth));
+    }
     /**
      * Appends one open row to the `network_outages` journal.
      *

package/lib/archive/create-adjunct-tables.js

@@ -293,6 +293,7 @@ export async function createAdjunctTables(instance) {
             t.integer('new_pages').nullable();
             t.integer('new_resources').nullable();
             t.integer('scope_skipped').nullable();
+            t.integer('exclude_skipped').nullable();
             t.integer('invalid_skipped').nullable();
             t.text('notes').nullable();
             t.index('ran_at');

package/lib/archive/create-entity-tables.js

@@ -198,6 +198,7 @@ export async function createEntityTables(instance) {
 			header_set_id    INTEGER REFERENCES header_sets(id),
 			redirect_dest_id INTEGER REFERENCES content_items(id) DEFERRABLE INITIALLY DEFERRED,
 			alias_of_id      INTEGER REFERENCES content_items(id) DEFERRABLE INITIALLY DEFERRED,
+			dedupe_cap_event_id INTEGER REFERENCES dedupe_cap_events(id) DEFERRABLE INITIALLY DEFERRED,
 			source           TEXT NOT NULL DEFAULT 'crawled',
 			first_crawled_at INTEGER,
 			last_crawled_at  INTEGER,
@@ -218,6 +219,15 @@ export async function createEntityTables(instance) {
     // `migrateContentItemsAliasOfId` instead, which runs after the
     // column-add guard for both fresh and legacy archives (same reasoning as
     // `page_meta.body_hash`'s index).
+    //
+    // `dedupe_cap_event_id` gets no index anywhere, not even in its own
+    // migration (`migrateContentItemsDedupeCapEventId`) — unlike
+    // `alias_of_id`, there is no known hot read path filtering on this
+    // column yet (`--dedupe-cap` is opt-in and the marked row count is
+    // small: capped shapes × matching URLs). Adding a speculative index
+    // without a measured query to justify it violates this archive's
+    // "no speculative index" rule; add one later with `EXPLAIN QUERY PLAN`
+    // evidence if a real hot path emerges.
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_content_type_id ON content_items(content_type_id)');
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_crawl_order ON content_items(crawl_order)');
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_source ON content_items(source)');

package/lib/archive/database.d.ts

@@ -310,6 +310,15 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @param urls - URL strings already in `withoutHashAndAuth` form.
      */
     insertInventorySeeds(urls: readonly string[]): Promise<void>;
+    /**
+     * Records exclude-matched inventory URLs as terminal skipped pages
+     * (`scraped=1`, `is_skipped=1`, `skip_reason='excluded'`,
+     * `source='inventory-seed'`). Delegates to
+     * {@link insertInventorySkippedPagesOp} — see that op's JSDoc for the
+     * normal-crawl parity rationale.
+     * @param urls - URL strings already in `withoutHashAndAuth` form.
+     */
+    insertInventorySkippedPages(urls: readonly string[]): Promise<void>;
     /**
      * Appends one open (`ended_at = NULL`) row to the `network_outages`
      * journal. Delegates to {@link insertNetworkOutageOp}.

package/lib/archive/database.js

@@ -52,6 +52,7 @@ import { getScrapedHtmlPageCount as getScrapedHtmlPageCountOp } from './db-ops/p
 import { repromoteExternalPages as repromoteExternalPagesOp } from './db-ops/pages/reset/repromote-external-pages.js';
 import { resetFailedPages as resetFailedPagesOp } from './db-ops/pages/reset/reset-failed-pages.js';
 import { insertInventorySeeds as insertInventorySeedsOp } from './db-ops/pages/write/insert-inventory-seeds.js';
+import { insertInventorySkippedPages as insertInventorySkippedPagesOp } from './db-ops/pages/write/insert-inventory-skipped-pages.js';
 import { recordRedirect as recordRedirectOp } from './db-ops/pages/write/record-redirect.js';
 import { setSkippedPage as setSkippedPageOp } from './db-ops/pages/write/set-skipped-page.js';
 import { updatePage as updatePageOp } from './db-ops/pages/write/update-page.js';
@@ -473,6 +474,17 @@ export class Database extends EventEmitter {
     async insertInventorySeeds(urls) {
         return emitErrorAndRetry(this, 'Database.insertInventorySeeds', async () => await insertInventorySeedsOp(this.#instance, this.#writeRefCaches, urls), retrySetting);
     }
+    /**
+     * Records exclude-matched inventory URLs as terminal skipped pages
+     * (`scraped=1`, `is_skipped=1`, `skip_reason='excluded'`,
+     * `source='inventory-seed'`). Delegates to
+     * {@link insertInventorySkippedPagesOp} — see that op's JSDoc for the
+     * normal-crawl parity rationale.
+     * @param urls - URL strings already in `withoutHashAndAuth` form.
+     */
+    async insertInventorySkippedPages(urls) {
+        return emitErrorAndRetry(this, 'Database.insertInventorySkippedPages', async () => await insertInventorySkippedPagesOp(this.#instance, this.#writeRefCaches, urls), retrySetting);
+    }
     /**
      * Appends one open (`ended_at = NULL`) row to the `network_outages`
      * journal. Delegates to {@link insertNetworkOutageOp}.

package/lib/archive/db-ops/config/get-config.js

@@ -19,6 +19,7 @@ export async function getConfig(knex) {
         excludeUrls: getJSON(config.excludeUrls, []),
         roots: getJSON(config.roots, []),
         retry: config.retry ?? 3,
+        maxExcludedDepth: config.maxExcludedDepth ?? 0,
     };
     // @ts-expect-error — `id` is the primary key, not part of the public Config shape
     delete opt.id;

package/lib/archive/db-ops/inventory/record-inventory-run.js

@@ -27,6 +27,7 @@ export async function recordInventoryRun(knex, meta) {
         new_pages: meta.new_pages ?? null,
         new_resources: meta.new_resources ?? null,
         scope_skipped: meta.scope_skipped ?? null,
+        exclude_skipped: meta.exclude_skipped ?? null,
         invalid_skipped: meta.invalid_skipped ?? null,
         notes: meta.notes ?? null,
     })

package/lib/archive/db-ops/lifecycle/init.d.ts

@@ -2,7 +2,8 @@ import type { Knex } from 'knex';
 /**
  * Initializes the database schema if tables do not exist, then runs the
  * remaining lightweight migrations (`info.roots`, `info.mainContentSelector`,
- * `page_meta.main_content_*`, `inventory_runs.invalid_skipped`).
+ * `page_meta.main_content_*`, `inventory_runs.invalid_skipped`,
+ * `inventory_runs.exclude_skipped`).
  *
  * There is deliberately no per-table *table-creation* migration chain here:
  * `assertCompatibleVersion` (called below, before any schema work) rejects
@@ -19,7 +20,8 @@ import type { Knex } from 'knex';
  * needs an explicit `hasColumn`-guarded `ALTER TABLE` here (`migrateInfoRoots`,
  * `migrateMainContentsColumns`, `migratePageMetaBodyHash`,
  * `migratePageMetaConsoleErrorCount`, `migrateContentItemsAliasOfId`,
- * `migrateInventoryRunsInvalidSkipped`) rather than a DDL-string change alone.
+ * `migrateContentItemsDedupeCapEventId`, `migrateInventoryRunsInvalidSkipped`,
+ * `migrateInventoryRunsExcludeSkipped`) rather than a DDL-string change alone.
  *
  * `closeStaleOpenNetworkOutages` is not a schema migration (no columns
  * change) but belongs at this same boot phase for the same reason the

package/lib/archive/db-ops/lifecycle/init.js

@@ -1,8 +1,10 @@
 import { applyConnectionPragmas, initSchema } from '../../init-schema.js';
 import { assertCompatibleVersion } from '../../meta/assert-compatible-version.js';
 import { migrateContentItemsAliasOfId } from '../../migrate-content-items-alias-of-id.js';
+import { migrateContentItemsDedupeCapEventId } from '../../migrate-content-items-dedupe-cap-event-id.js';
 import { migrateInfoMainContentSelector } from '../../migrate-info-main-content-selector.js';
 import { migrateInfoRoots } from '../../migrate-info-roots.js';
+import { migrateInventoryRunsExcludeSkipped } from '../../migrate-inventory-runs-exclude-skipped.js';
 import { migrateInventoryRunsInvalidSkipped } from '../../migrate-inventory-runs-invalid-skipped.js';
 import { migrateMainContentsColumns } from '../../migrate-main-contents-columns.js';
 import { migratePageMetaBodyHash } from '../../migrate-page-meta-body-hash.js';
@@ -11,7 +13,8 @@ import { closeStaleOpenNetworkOutages } from '../outages/close-stale-open-networ
 /**
  * Initializes the database schema if tables do not exist, then runs the
  * remaining lightweight migrations (`info.roots`, `info.mainContentSelector`,
- * `page_meta.main_content_*`, `inventory_runs.invalid_skipped`).
+ * `page_meta.main_content_*`, `inventory_runs.invalid_skipped`,
+ * `inventory_runs.exclude_skipped`).
  *
  * There is deliberately no per-table *table-creation* migration chain here:
  * `assertCompatibleVersion` (called below, before any schema work) rejects
@@ -28,7 +31,8 @@ import { closeStaleOpenNetworkOutages } from '../outages/close-stale-open-networ
  * needs an explicit `hasColumn`-guarded `ALTER TABLE` here (`migrateInfoRoots`,
  * `migrateMainContentsColumns`, `migratePageMetaBodyHash`,
  * `migratePageMetaConsoleErrorCount`, `migrateContentItemsAliasOfId`,
- * `migrateInventoryRunsInvalidSkipped`) rather than a DDL-string change alone.
+ * `migrateContentItemsDedupeCapEventId`, `migrateInventoryRunsInvalidSkipped`,
+ * `migrateInventoryRunsExcludeSkipped`) rather than a DDL-string change alone.
  *
  * `closeStaleOpenNetworkOutages` is not a schema migration (no columns
  * change) but belongs at this same boot phase for the same reason the
@@ -66,6 +70,12 @@ export async function init(knex, readOnly) {
     await migratePageMetaBodyHash(knex);
     await migratePageMetaConsoleErrorCount(knex);
     await migrateContentItemsAliasOfId(knex);
+    // Runs after `initSchema` above, which already created
+    // `dedupe_cap_events` (an adjunct table) unconditionally — so the new
+    // column's `REFERENCES dedupe_cap_events(id)` target always exists by
+    // this point, for both fresh and legacy archives.
+    await migrateContentItemsDedupeCapEventId(knex);
     await migrateInventoryRunsInvalidSkipped(knex);
+    await migrateInventoryRunsExcludeSkipped(knex);
     await closeStaleOpenNetworkOutages(knex);
 }

package/lib/archive/db-ops/pages/write/insert-inventory-content-items.d.ts

@@ -0,0 +1,38 @@
+import type { WriteRefCaches } from '../../_shared/types.js';
+import type { Knex } from 'knex';
+/**
+ * Parameters for {@link insertInventoryContentItems}.
+ */
+export interface InsertInventoryContentItemsParams {
+    /** Knex query builder connected to the archive DB. */
+    readonly knex: Knex;
+    /** The connection's write-side id caches. */
+    readonly caches: WriteRefCaches;
+    /** URL strings already in `withoutHashAndAuth` form. */
+    readonly urls: readonly string[];
+    /** `content_items` column values shared by every inserted row (everything except `url_id`). */
+    readonly row: Readonly<Record<string, number | string>>;
+    /** Calling op's name, used to prefix the unresolved-url_ref error message. */
+    readonly opName: string;
+}
+/**
+ * Shared body of the inventory `content_items` bulk-insert ops
+ * (`insertInventorySeeds` / `insertInventorySkippedPages`): chunked
+ * `url_refs` upsert → id resolution → `content_items` insert-ignore →
+ * write-cache population. The two callers differ only in the row
+ * constants they stamp on every row, so the invariant-heavy plumbing
+ * lives here exactly once:
+ *
+ * - Chunked into 500-URL batches so SQLite's bound-parameter limit
+ *   (`SQLITE_MAX_VARIABLE_NUMBER`) cannot be hit even on a
+ *   tens-of-thousands inventory list.
+ * - Both inserts are `ON CONFLICT ... IGNORE`, so existing rows — in
+ *   particular previously crawled pages — are never overwritten
+ *   (crawled-wins), and within-list duplicates collapse to one row.
+ * - The `urlIds` / `contentItems` write caches are populated from what
+ *   the DB actually holds after the insert (not from the attempted row
+ *   values), keeping later cache-hits consistent with conflict-ignored
+ *   rows.
+ * @param params - See {@link InsertInventoryContentItemsParams}.
+ */
+export declare function insertInventoryContentItems(params: InsertInventoryContentItemsParams): Promise<void>;

package/lib/archive/db-ops/pages/write/insert-inventory-content-items.js

@@ -0,0 +1,59 @@
+import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+import { resolveUrlRefs } from '../../../populate-entity-tables/resolve-url-refs.js';
+import { decomposeUrl } from '../../../populate-ref-tables/decompose-url.js';
+/**
+ * Shared body of the inventory `content_items` bulk-insert ops
+ * (`insertInventorySeeds` / `insertInventorySkippedPages`): chunked
+ * `url_refs` upsert → id resolution → `content_items` insert-ignore →
+ * write-cache population. The two callers differ only in the row
+ * constants they stamp on every row, so the invariant-heavy plumbing
+ * lives here exactly once:
+ *
+ * - Chunked into 500-URL batches so SQLite's bound-parameter limit
+ *   (`SQLITE_MAX_VARIABLE_NUMBER`) cannot be hit even on a
+ *   tens-of-thousands inventory list.
+ * - Both inserts are `ON CONFLICT ... IGNORE`, so existing rows — in
+ *   particular previously crawled pages — are never overwritten
+ *   (crawled-wins), and within-list duplicates collapse to one row.
+ * - The `urlIds` / `contentItems` write caches are populated from what
+ *   the DB actually holds after the insert (not from the attempted row
+ *   values), keeping later cache-hits consistent with conflict-ignored
+ *   rows.
+ * @param params - See {@link InsertInventoryContentItemsParams}.
+ */
+export async function insertInventoryContentItems(params) {
+    const { knex, caches, urls, row, opName } = params;
+    if (urls.length === 0) {
+        return;
+    }
+    await eachSplitted([...urls], 500, async (chunk) => {
+        await knex('url_refs')
+            .insert(chunk.map((url) => ({ url, ...decomposeUrl(url) })))
+            .onConflict('url')
+            .ignore();
+        const urlIds = await resolveUrlRefs(knex, chunk);
+        const rows = chunk.map((url) => {
+            const urlId = urlIds.get(url);
+            if (urlId === undefined) {
+                throw new Error(`${opName}: url_refs.id not resolved for ${url}`);
+            }
+            caches.urlIds.set(url, urlId);
+            return {
+                url_id: urlId,
+                ...row,
+            };
+        });
+        await knex('content_items').insert(rows).onConflict('url_id').ignore();
+        const inserted = (await knex
+            .select('ci.id', 'ci.source', 'ur.url')
+            .from('content_items as ci')
+            .join('url_refs as ur', 'ur.id', 'ci.url_id')
+            .whereIn('ur.url', chunk));
+        for (const insertedRow of inserted) {
+            caches.contentItems.set(insertedRow.url, {
+                id: insertedRow.id,
+                source: insertedRow.source,
+            });
+        }
+    });
+}

package/lib/archive/db-ops/pages/write/insert-inventory-seeds.d.ts

@@ -23,15 +23,14 @@ import type { Knex } from 'knex';
  * behaviour (a seed that turned out to be reachable is not an orphan
  * and should not retain the inventory label).
  *
- * Chunked into 500-URL batches so SQLite's bound-parameter limit
- * (`SQLITE_MAX_VARIABLE_NUMBER`) cannot be hit even on a
- * tens-of-thousands inventory list.
- *
  * Called by `CrawlerOrchestrator.inventory` during the
  * `.bak`-protected ingestion phase, so any failure here aborts the run
- * and restores from backup — the operator reruns from scratch.
+ * and restores from backup — the operator reruns from scratch. The
+ * chunking / conflict-ignore / cache-population plumbing lives in
+ * {@link insertInventoryContentItems}, shared with
+ * `insertInventorySkippedPages`.
  * @param knex - Knex query builder connected to the archive DB.
- * @param caches
+ * @param caches - The connection's write-side id caches.
  * @param urls - URL strings already in `withoutHashAndAuth` form.
  */
 export declare function insertInventorySeeds(knex: Knex, caches: WriteRefCaches, urls: readonly string[]): Promise<void>;

package/lib/archive/db-ops/pages/write/insert-inventory-seeds.js

@@ -1,6 +1,4 @@
-import { eachSplitted } from '../../../../utils/array/each-splitted.js';
-import { resolveUrlRefs } from '../../../populate-entity-tables/resolve-url-refs.js';
-import { decomposeUrl } from '../../../populate-ref-tables/decompose-url.js';
+import { insertInventoryContentItems } from './insert-inventory-content-items.js';
 /**
  * Pre-insert inventory HTML seeds into `content_items` as `scraped = 0`,
  * `source = 'inventory-seed'` placeholders so the URL's existence in the
@@ -24,49 +22,27 @@ import { decomposeUrl } from '../../../populate-ref-tables/decompose-url.js';
  * behaviour (a seed that turned out to be reachable is not an orphan
  * and should not retain the inventory label).
  *
- * Chunked into 500-URL batches so SQLite's bound-parameter limit
- * (`SQLITE_MAX_VARIABLE_NUMBER`) cannot be hit even on a
- * tens-of-thousands inventory list.
- *
  * Called by `CrawlerOrchestrator.inventory` during the
  * `.bak`-protected ingestion phase, so any failure here aborts the run
- * and restores from backup — the operator reruns from scratch.
+ * and restores from backup — the operator reruns from scratch. The
+ * chunking / conflict-ignore / cache-population plumbing lives in
+ * {@link insertInventoryContentItems}, shared with
+ * `insertInventorySkippedPages`.
  * @param knex - Knex query builder connected to the archive DB.
- * @param caches
+ * @param caches - The connection's write-side id caches.
  * @param urls - URL strings already in `withoutHashAndAuth` form.
  */
 export async function insertInventorySeeds(knex, caches, urls) {
-    if (urls.length === 0) {
-        return;
-    }
-    await eachSplitted([...urls], 500, async (chunk) => {
-        await knex('url_refs')
-            .insert(chunk.map((url) => ({ url, ...decomposeUrl(url) })))
-            .onConflict('url')
-            .ignore();
-        const urlIds = await resolveUrlRefs(knex, chunk);
-        const rows = chunk.map((url) => {
-            const urlId = urlIds.get(url);
-            if (urlId === undefined) {
-                throw new Error(`insertInventorySeeds: url_refs.id not resolved for ${url}`);
-            }
-            caches.urlIds.set(url, urlId);
-            return {
-                url_id: urlId,
-                scraped: 0,
-                is_external: 0,
-                is_target: 0,
-                source: 'inventory-seed',
-            };
-        });
-        await knex('content_items').insert(rows).onConflict('url_id').ignore();
-        const inserted = (await knex
-            .select('ci.id', 'ci.source', 'ur.url')
-            .from('content_items as ci')
-            .join('url_refs as ur', 'ur.id', 'ci.url_id')
-            .whereIn('ur.url', chunk));
-        for (const row of inserted) {
-            caches.contentItems.set(row.url, { id: row.id, source: row.source });
-        }
+    await insertInventoryContentItems({
+        knex,
+        caches,
+        urls,
+        row: {
+            scraped: 0,
+            is_external: 0,
+            is_target: 0,
+            source: 'inventory-seed',
+        },
+        opName: 'insertInventorySeeds',
     });
 }

package/lib/archive/db-ops/pages/write/insert-inventory-skipped-pages.d.ts

@@ -0,0 +1,42 @@
+import type { WriteRefCaches } from '../../_shared/types.js';
+import type { Knex } from 'knex';
+/**
+ * Record exclude-matched inventory URLs into `content_items` as
+ * `scraped = 1`, `is_skipped = 1`, `skip_reason = 'excluded'`,
+ * `source = 'inventory-seed'` rows — the same terminal state the normal
+ * crawl's fetch-time `shouldSkipUrl` gate produces via `setSkippedPage`
+ * for link-discovered excluded URLs.
+ *
+ * Why a dedicated write path instead of routing these URLs through the
+ * crawler's gate: non-HTML inventory URLs never enter the crawler at all
+ * (they are recorded straight into `resources`), so the gate cannot see
+ * them, and pre-inserting HTML seeds only to have the dealer skip them
+ * wastes dealer slots for a verdict already known at ingestion time.
+ * Writing the terminal skipped state directly keeps the invariant "the
+ * same URL lands in the same archive state regardless of how it was
+ * discovered (anchor vs inventory list)" for both classifications
+ * (issue #260).
+ *
+ * `scraped = 1` is load-bearing: it keeps these rows out of
+ * `getCrawlingState`'s strict pending set, so `--resume` after an
+ * interrupted inventory pass does not try to fetch operator-excluded
+ * URLs.
+ *
+ * Idempotent: both the `url_refs` and `content_items` inserts are
+ * `ON CONFLICT ... IGNORE`, so an existing row — in particular a
+ * previously crawled page that now matches the exclusion config — is
+ * never downgraded to skipped by this path (crawled-wins). The
+ * orchestrator additionally filters known URLs out before calling this,
+ * so conflicts here are limited to within-list duplicates.
+ *
+ * Called by `CrawlerOrchestrator.inventory` during the `.bak`-protected
+ * ingestion phase, so any failure here aborts the run and restores from
+ * backup — the operator reruns from scratch. The chunking /
+ * conflict-ignore / cache-population plumbing lives in
+ * {@link insertInventoryContentItems}, shared with
+ * `insertInventorySeeds`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param urls - URL strings already in `withoutHashAndAuth` form.
+ */
+export declare function insertInventorySkippedPages(knex: Knex, caches: WriteRefCaches, urls: readonly string[]): Promise<void>;

package/lib/archive/db-ops/pages/write/insert-inventory-skipped-pages.js

@@ -0,0 +1,56 @@
+import { insertInventoryContentItems } from './insert-inventory-content-items.js';
+/**
+ * Record exclude-matched inventory URLs into `content_items` as
+ * `scraped = 1`, `is_skipped = 1`, `skip_reason = 'excluded'`,
+ * `source = 'inventory-seed'` rows — the same terminal state the normal
+ * crawl's fetch-time `shouldSkipUrl` gate produces via `setSkippedPage`
+ * for link-discovered excluded URLs.
+ *
+ * Why a dedicated write path instead of routing these URLs through the
+ * crawler's gate: non-HTML inventory URLs never enter the crawler at all
+ * (they are recorded straight into `resources`), so the gate cannot see
+ * them, and pre-inserting HTML seeds only to have the dealer skip them
+ * wastes dealer slots for a verdict already known at ingestion time.
+ * Writing the terminal skipped state directly keeps the invariant "the
+ * same URL lands in the same archive state regardless of how it was
+ * discovered (anchor vs inventory list)" for both classifications
+ * (issue #260).
+ *
+ * `scraped = 1` is load-bearing: it keeps these rows out of
+ * `getCrawlingState`'s strict pending set, so `--resume` after an
+ * interrupted inventory pass does not try to fetch operator-excluded
+ * URLs.
+ *
+ * Idempotent: both the `url_refs` and `content_items` inserts are
+ * `ON CONFLICT ... IGNORE`, so an existing row — in particular a
+ * previously crawled page that now matches the exclusion config — is
+ * never downgraded to skipped by this path (crawled-wins). The
+ * orchestrator additionally filters known URLs out before calling this,
+ * so conflicts here are limited to within-list duplicates.
+ *
+ * Called by `CrawlerOrchestrator.inventory` during the `.bak`-protected
+ * ingestion phase, so any failure here aborts the run and restores from
+ * backup — the operator reruns from scratch. The chunking /
+ * conflict-ignore / cache-population plumbing lives in
+ * {@link insertInventoryContentItems}, shared with
+ * `insertInventorySeeds`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param urls - URL strings already in `withoutHashAndAuth` form.
+ */
+export async function insertInventorySkippedPages(knex, caches, urls) {
+    await insertInventoryContentItems({
+        knex,
+        caches,
+        urls,
+        row: {
+            scraped: 1,
+            is_external: 0,
+            is_target: 0,
+            is_skipped: 1,
+            skip_reason: 'excluded',
+            source: 'inventory-seed',
+        },
+        opName: 'insertInventorySkippedPages',
+    });
+}

package/lib/archive/migrate-content-items-dedupe-cap-event-id.d.ts

@@ -0,0 +1,41 @@
+import type { Knex } from 'knex';
+/**
+ * Adds the `content_items.dedupe_cap_event_id` column to archives created
+ * before this feature.
+ *
+ * `content_items` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `content_items` predates this change. This
+ * mirrors {@link import('./migrate-content-items-alias-of-id.js').migrateContentItemsAliasOfId}'s
+ * catch-up: a `hasColumn`-guarded `ALTER TABLE` for the one column
+ * `CREATE TABLE IF NOT EXISTS` cannot retrofit.
+ *
+ * Uses a raw `ALTER TABLE` (not the knex schema builder) so the retrofitted
+ * column's `REFERENCES dedupe_cap_events(id) DEFERRABLE INITIALLY DEFERRED`
+ * constraint matches the fresh-archive DDL bit-for-bit.
+ *
+ * Unlike `migrateContentItemsAliasOfId`, this migration never creates an
+ * index for the column — `--dedupe-cap` is opt-in and the number of rows a
+ * cap event ever marks is small (capped shapes × matching URLs), so there is
+ * no measured hot path to justify one. See `createEntityTables`'s DDL
+ * comment for the same reasoning.
+ *
+ * Only adds the column — it does not compute values for existing rows (they
+ * stay `NULL`). That computation runs separately, from
+ * `backfillDedupeCapEventId` during a viewer-read-model build, since it
+ * requires recomputing `computeShapeKey` against every internal page's URL
+ * and matching it against `dedupe_cap_events.shape_key`.
+ *
+ * Idempotent: adding the column is a no-op once it exists. Guards on
+ * `content_items`'s existence defensively, though by the time this runs
+ * (after `initSchema`, itself after `assertCompatibleVersion` rejects
+ * pre-0.13 archives) the table is always present.
+ * @param instance - The Knex query builder instance connected to the database.
+ * @example
+ * ```ts
+ * await migrateContentItemsDedupeCapEventId(knex);
+ * ```
+ */
+export declare function migrateContentItemsDedupeCapEventId(instance: Knex): Promise<void>;

package/lib/archive/migrate-content-items-dedupe-cap-event-id.js

@@ -0,0 +1,51 @@
+/**
+ * Adds the `content_items.dedupe_cap_event_id` column to archives created
+ * before this feature.
+ *
+ * `content_items` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `content_items` predates this change. This
+ * mirrors {@link import('./migrate-content-items-alias-of-id.js').migrateContentItemsAliasOfId}'s
+ * catch-up: a `hasColumn`-guarded `ALTER TABLE` for the one column
+ * `CREATE TABLE IF NOT EXISTS` cannot retrofit.
+ *
+ * Uses a raw `ALTER TABLE` (not the knex schema builder) so the retrofitted
+ * column's `REFERENCES dedupe_cap_events(id) DEFERRABLE INITIALLY DEFERRED`
+ * constraint matches the fresh-archive DDL bit-for-bit.
+ *
+ * Unlike `migrateContentItemsAliasOfId`, this migration never creates an
+ * index for the column — `--dedupe-cap` is opt-in and the number of rows a
+ * cap event ever marks is small (capped shapes × matching URLs), so there is
+ * no measured hot path to justify one. See `createEntityTables`'s DDL
+ * comment for the same reasoning.
+ *
+ * Only adds the column — it does not compute values for existing rows (they
+ * stay `NULL`). That computation runs separately, from
+ * `backfillDedupeCapEventId` during a viewer-read-model build, since it
+ * requires recomputing `computeShapeKey` against every internal page's URL
+ * and matching it against `dedupe_cap_events.shape_key`.
+ *
+ * Idempotent: adding the column is a no-op once it exists. Guards on
+ * `content_items`'s existence defensively, though by the time this runs
+ * (after `initSchema`, itself after `assertCompatibleVersion` rejects
+ * pre-0.13 archives) the table is always present.
+ * @param instance - The Knex query builder instance connected to the database.
+ * @example
+ * ```ts
+ * await migrateContentItemsDedupeCapEventId(knex);
+ * ```
+ */
+export async function migrateContentItemsDedupeCapEventId(instance) {
+    const hasContentItems = await instance.schema.hasTable('content_items');
+    if (!hasContentItems) {
+        return;
+    }
+    const hasColumn = await instance.schema.hasColumn('content_items', 'dedupe_cap_event_id');
+    if (!hasColumn) {
+        await instance.raw('ALTER TABLE content_items ADD COLUMN dedupe_cap_event_id INTEGER REFERENCES dedupe_cap_events(id) DEFERRABLE INITIALLY DEFERRED');
+        // eslint-disable-next-line no-console
+        console.error('[migrate] content_items.dedupe_cap_event_id column added');
+    }
+}

package/lib/archive/migrate-entity-tables.d.ts

@@ -29,6 +29,16 @@ import type { Knex } from 'knex';
  * `scripts/migrate-to-0.13.mjs` orders the two calls statically so this
  * ordering is enforced there, not here.
  *
+ * `content_items` also declares `dedupe_cap_event_id REFERENCES
+ * dedupe_cap_events(id)` — an adjunct table (`createAdjunctTables`), not a
+ * ref table. This function only creates the empty tables (schema-only, see
+ * above), so it does not itself need `dedupe_cap_events` to exist. Callers
+ * that write data into `content_items` afterward do: under `PRAGMA
+ * foreign_keys = ON`, SQLite refuses to even prepare an INSERT/UPDATE
+ * against a table with an unresolvable `REFERENCES` target, so
+ * `scripts/migrate-to-0.13.mjs` creates adjunct tables immediately after
+ * this migration and before any entity-table data write.
+ *
  * **Idempotency**: `createEntityTables` itself uses
  * `CREATE TABLE IF NOT EXISTS` / `CREATE INDEX IF NOT EXISTS` for every
  * statement, so calling it multiple times against any DB state is safe.

package/lib/archive/migrate-entity-tables.js

@@ -29,6 +29,16 @@ import { createEntityTables } from './create-entity-tables.js';
  * `scripts/migrate-to-0.13.mjs` orders the two calls statically so this
  * ordering is enforced there, not here.
  *
+ * `content_items` also declares `dedupe_cap_event_id REFERENCES
+ * dedupe_cap_events(id)` — an adjunct table (`createAdjunctTables`), not a
+ * ref table. This function only creates the empty tables (schema-only, see
+ * above), so it does not itself need `dedupe_cap_events` to exist. Callers
+ * that write data into `content_items` afterward do: under `PRAGMA
+ * foreign_keys = ON`, SQLite refuses to even prepare an INSERT/UPDATE
+ * against a table with an unresolvable `REFERENCES` target, so
+ * `scripts/migrate-to-0.13.mjs` creates adjunct tables immediately after
+ * this migration and before any entity-table data write.
+ *
  * **Idempotency**: `createEntityTables` itself uses
  * `CREATE TABLE IF NOT EXISTS` / `CREATE INDEX IF NOT EXISTS` for every
  * statement, so calling it multiple times against any DB state is safe.

package/lib/archive/migrate-inventory-runs-exclude-skipped.d.ts

@@ -0,0 +1,20 @@
+import type { Knex } from 'knex';
+/**
+ * Adds the `inventory_runs.exclude_skipped` column to archives created
+ * before it existed. `CREATE TABLE IF NOT EXISTS` (used for `inventory_runs`
+ * itself) cannot retrofit a new column onto an already-existing table, so
+ * this lightweight, `hasColumn`-guarded `ALTER TABLE` runs on every
+ * `initSchema` call — idempotent, and self-healing for archives whose
+ * provisioning crashed partway through.
+ *
+ * Pre-migration rows stay `NULL`: those runs predate ingestion-side
+ * exclusion (issue #260), so their excluded URLs were imported as real
+ * pages/resources rather than recorded as skipped — `NULL` means "not
+ * measured", not `0`.
+ *
+ * The column is pure audit output: written once per run and read back
+ * only by `listInventoryRuns` display surfaces, never consumed by any
+ * runtime decision — matching `scope_skipped` / `invalid_skipped`.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function migrateInventoryRunsExcludeSkipped(instance: Knex): Promise<void>;

package/lib/archive/migrate-inventory-runs-exclude-skipped.js

@@ -0,0 +1,33 @@
+/**
+ * Adds the `inventory_runs.exclude_skipped` column to archives created
+ * before it existed. `CREATE TABLE IF NOT EXISTS` (used for `inventory_runs`
+ * itself) cannot retrofit a new column onto an already-existing table, so
+ * this lightweight, `hasColumn`-guarded `ALTER TABLE` runs on every
+ * `initSchema` call — idempotent, and self-healing for archives whose
+ * provisioning crashed partway through.
+ *
+ * Pre-migration rows stay `NULL`: those runs predate ingestion-side
+ * exclusion (issue #260), so their excluded URLs were imported as real
+ * pages/resources rather than recorded as skipped — `NULL` means "not
+ * measured", not `0`.
+ *
+ * The column is pure audit output: written once per run and read back
+ * only by `listInventoryRuns` display surfaces, never consumed by any
+ * runtime decision — matching `scope_skipped` / `invalid_skipped`.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function migrateInventoryRunsExcludeSkipped(instance) {
+    const hasTable = await instance.schema.hasTable('inventory_runs');
+    if (!hasTable) {
+        return;
+    }
+    const hasColumn = await instance.schema.hasColumn('inventory_runs', 'exclude_skipped');
+    if (hasColumn) {
+        return;
+    }
+    await instance.schema.table('inventory_runs', (t) => {
+        t.integer('exclude_skipped');
+    });
+    // eslint-disable-next-line no-console
+    console.error('[migrate] inventory_runs.exclude_skipped column added');
+}

package/lib/archive/populate-entity-tables/test-utils/setup-entities-db.d.ts

@@ -8,6 +8,13 @@ import knex from 'knex';
  *   actually read.
  * - The 0.13 ref / header tables (via {@link createRefTables}).
  * - The 0.13 entity tables (via {@link createEntityTables}).
+ * - The 0.13 adjunct tables (via {@link createAdjunctTables}) — required
+ *   because `content_items.dedupe_cap_event_id REFERENCES
+ *   dedupe_cap_events(id)`; under `PRAGMA foreign_keys = ON` (enabled
+ *   below), inserting into `content_items` fails with `no such table:
+ *   dedupe_cap_events` if the adjunct tables were skipped. `initSchema`
+ *   always calls both create functions together, so this mirrors a real
+ *   archive's actual schema rather than an artificially incomplete one.
  *
  * Every 0.13 populate spec calls this to obtain a fresh DB. The
  * caller is responsible for `db.destroy()` (spec `afterEach`).

package/lib/archive/populate-entity-tables/test-utils/setup-entities-db.js

@@ -1,4 +1,5 @@
 import knex from 'knex';
+import { createAdjunctTables } from '../../create-adjunct-tables.js';
 import { createEntityTables } from '../../create-entity-tables.js';
 import { createRefTables } from '../../create-ref-tables.js';
 import { LibsqlDialect } from '../../libsql-dialect.js';
@@ -11,6 +12,13 @@ import { LibsqlDialect } from '../../libsql-dialect.js';
  *   actually read.
  * - The 0.13 ref / header tables (via {@link createRefTables}).
  * - The 0.13 entity tables (via {@link createEntityTables}).
+ * - The 0.13 adjunct tables (via {@link createAdjunctTables}) — required
+ *   because `content_items.dedupe_cap_event_id REFERENCES
+ *   dedupe_cap_events(id)`; under `PRAGMA foreign_keys = ON` (enabled
+ *   below), inserting into `content_items` fails with `no such table:
+ *   dedupe_cap_events` if the adjunct tables were skipped. `initSchema`
+ *   always calls both create functions together, so this mirrors a real
+ *   archive's actual schema rather than an artificially incomplete one.
  *
  * Every 0.13 populate spec calls this to obtain a fresh DB. The
  * caller is responsible for `db.destroy()` (spec `afterEach`).
@@ -174,5 +182,6 @@ export async function setupMigrationDb() {
 	`);
     await createRefTables(db);
     await createEntityTables(db);
+    await createAdjunctTables(db);
     return db;
 }

package/lib/archive/types.d.ts

@@ -121,6 +121,7 @@ export type PageSource = 'crawled' | 'inventory-seed' | 'inventory-discovered';
  *   new_pages: 1234,
  *   new_resources: 56,
  *   scope_skipped: 7,
+ *   exclude_skipped: 3,
  * });
  */
 export interface InventoryRunMeta {
@@ -138,6 +139,8 @@ export interface InventoryRunMeta {
     new_resources?: number | null;
     /** Number of input URLs dropped because they fell outside the archived scope. */
     scope_skipped?: number | null;
+    /** Number of novel in-scope input URLs recorded as terminal skipped pages (`is_skipped=1`, `skip_reason='excluded'`) instead of being imported, because they matched the effective `excludes` / `excludeUrls` config. Pure audit output like every other count on this row — written once per run, read back only by `listInventoryRuns` display surfaces, never consumed by any runtime decision. `null` on rows written before the column existed (those runs predate ingestion-side exclusion — their excluded URLs were imported as real pages/resources, not counted). */
+    exclude_skipped?: number | null;
     /** Number of source-file lines dropped by the CLI for failing URL validation, before this row's `total_lines` was counted. `null` for programmatic callers that built the URL list in-memory (no source file to have invalid lines). */
     invalid_skipped?: number | null;
     /** Free-form text for backfill annotations or operator notes. */

package/lib/crawler-orchestrator.d.ts

@@ -228,9 +228,10 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * 1. Open the archive (writer mode, takes the archive lock).
      * 2. Reject list-mode archives — they hold metadata-only rows that
      *    inventory has no business touching.
-     * 3. Reject archives with unfinished `pending` URLs — those would inherit
-     *    the inventory `source` label by mistake. Operator must resume /
-     *    retry-failed first.
+     * 3. Warn (but proceed) on archives with unfinished `pending` URLs —
+     *    crawled-wins source priority keeps their labels stable; the
+     *    operator can `--resume` first if they want the prior work
+     *    finalized.
      * 4. If `source` is given, archive its exact bytes under
      *    `inventory/<sha256>.txt` (see {@link Archive.saveInventorySourceList}).
      *    Done before scope classification so even a run that discards every
@@ -243,17 +244,33 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * 6. Subtract URLs that already exist in `pages` or `resources` so the
      *    second (and N-th) inventory pass is a no-op for known rows — keeps
      *    `'inventory-seed'` rows from being silently demoted.
-     * 7. Make `<archive>.bak`. Anything thrown beyond this point restores
+     * 7. Split the remaining novel URLs on the effective `excludes` /
+     *    `excludeUrls` (archived config overlaid with this run's
+     *    overrides — the same inputs the crawl's fetch-time
+     *    `shouldSkipUrl` gate uses). Matching URLs are recorded as
+     *    terminal skipped pages (`is_skipped=1`,
+     *    `skip_reason='excluded'`, `source='inventory-seed'`) instead of
+     *    being imported — the same end state a link-discovered excluded
+     *    URL reaches in a normal crawl — and counted as
+     *    `exclude_skipped` (issue #260). Running this after step 6 keeps
+     *    previously crawled rows that newly match the exclusion config
+     *    untouched (crawled-wins). `excludeKeywords` does not
+     *    participate here: it matches rendered page content, which a URL
+     *    list does not have — HTML seeds still get it at render time via
+     *    the browser verdict.
+     * 8. Make `<archive>.bak`. Anything thrown beyond this point restores
      *    from the backup.
-     * 8. HEAD-probe each novel URL. Responses classified as HTML are queued
-     *    as Crawler seeds (`'inventory-seed'`); everything else is recorded
-     *    in `resources` directly as `'inventory-seed'` (no browser launch).
-     * 9. If any HTML seeds exist, start a Crawler with
+     * 9. Classify each importable novel URL by URL-extension heuristic
+     *    (no probe — see the in-body rationale). HTML-looking URLs are
+     *    queued as Crawler seeds (`'inventory-seed'`); everything else is
+     *    recorded in `resources` directly as `'inventory-seed'` (no
+     *    browser launch, no HEAD).
+     * 10. If any HTML seeds exist, start a Crawler with
      *    `inventoryMode = { seedUrls }` so the rendered page and every newly
      *    discovered downstream link is labelled correctly. `resume` is fed
      *    the existing `scraped` / `resources` sets so links into already-
      *    crawled pages stop at the seen-gate without re-rendering.
-     * 10. Drop the backup on success; restore it on any throw.
+     * 11. Drop the backup on success; restore it on any throw.
      *
      * Mutually exclusive with `--append` / `--retry-failed` / `--resume` /
      * `--diff` / `--list` / `--list-file` / `--single` / `--output` — the
@@ -276,7 +293,7 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      *   `inventoryUrls` in-memory; the audit row's `source_file_sha256`
      *   column will be `NULL` and no source list is archived.
      * @returns The orchestrator instance after a successful inventory pass.
-     * @throws {Error} When `inventoryUrls` is empty, the archive is in list mode, or pending URLs from a previous crawl remain unresolved.
+     * @throws {Error} When `inventoryUrls` is empty or the archive is in list mode. Unresolved pending URLs from a previous crawl do NOT throw — see step 3.
      */
     static inventory(archivePath: string, inventoryUrls: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback, source?: InventorySource | null): Promise<CrawlerOrchestrator>;
     /**

package/lib/crawler-orchestrator.js

@@ -16,6 +16,7 @@ import { isLikelyHtmlUrl } from './crawler/is-likely-html-url.js';
 import { networkOutageSummaryCounter } from './crawler/network-outage-summary-counter.js';
 import { PreloadShortCircuitError } from './crawler/preload-short-circuit-error.js';
 import { protocolAgnosticKey } from './crawler/protocol-agnostic-key.js';
+import { shouldSkipUrl } from './crawler/should-skip-url.js';
 import { crawlerLog, log } from './debug.js';
 import { normalizeToArray } from './normalize-to-array.js';
 import { resolveOutputPath } from './resolve-output-path.js';
@@ -658,9 +659,10 @@ export class CrawlerOrchestrator extends EventEmitter {
      * 1. Open the archive (writer mode, takes the archive lock).
      * 2. Reject list-mode archives — they hold metadata-only rows that
      *    inventory has no business touching.
-     * 3. Reject archives with unfinished `pending` URLs — those would inherit
-     *    the inventory `source` label by mistake. Operator must resume /
-     *    retry-failed first.
+     * 3. Warn (but proceed) on archives with unfinished `pending` URLs —
+     *    crawled-wins source priority keeps their labels stable; the
+     *    operator can `--resume` first if they want the prior work
+     *    finalized.
      * 4. If `source` is given, archive its exact bytes under
      *    `inventory/<sha256>.txt` (see {@link Archive.saveInventorySourceList}).
      *    Done before scope classification so even a run that discards every
@@ -673,17 +675,33 @@ export class CrawlerOrchestrator extends EventEmitter {
      * 6. Subtract URLs that already exist in `pages` or `resources` so the
      *    second (and N-th) inventory pass is a no-op for known rows — keeps
      *    `'inventory-seed'` rows from being silently demoted.
-     * 7. Make `<archive>.bak`. Anything thrown beyond this point restores
+     * 7. Split the remaining novel URLs on the effective `excludes` /
+     *    `excludeUrls` (archived config overlaid with this run's
+     *    overrides — the same inputs the crawl's fetch-time
+     *    `shouldSkipUrl` gate uses). Matching URLs are recorded as
+     *    terminal skipped pages (`is_skipped=1`,
+     *    `skip_reason='excluded'`, `source='inventory-seed'`) instead of
+     *    being imported — the same end state a link-discovered excluded
+     *    URL reaches in a normal crawl — and counted as
+     *    `exclude_skipped` (issue #260). Running this after step 6 keeps
+     *    previously crawled rows that newly match the exclusion config
+     *    untouched (crawled-wins). `excludeKeywords` does not
+     *    participate here: it matches rendered page content, which a URL
+     *    list does not have — HTML seeds still get it at render time via
+     *    the browser verdict.
+     * 8. Make `<archive>.bak`. Anything thrown beyond this point restores
      *    from the backup.
-     * 8. HEAD-probe each novel URL. Responses classified as HTML are queued
-     *    as Crawler seeds (`'inventory-seed'`); everything else is recorded
-     *    in `resources` directly as `'inventory-seed'` (no browser launch).
-     * 9. If any HTML seeds exist, start a Crawler with
+     * 9. Classify each importable novel URL by URL-extension heuristic
+     *    (no probe — see the in-body rationale). HTML-looking URLs are
+     *    queued as Crawler seeds (`'inventory-seed'`); everything else is
+     *    recorded in `resources` directly as `'inventory-seed'` (no
+     *    browser launch, no HEAD).
+     * 10. If any HTML seeds exist, start a Crawler with
      *    `inventoryMode = { seedUrls }` so the rendered page and every newly
      *    discovered downstream link is labelled correctly. `resume` is fed
      *    the existing `scraped` / `resources` sets so links into already-
      *    crawled pages stop at the seen-gate without re-rendering.
-     * 10. Drop the backup on success; restore it on any throw.
+     * 11. Drop the backup on success; restore it on any throw.
      *
      * Mutually exclusive with `--append` / `--retry-failed` / `--resume` /
      * `--diff` / `--list` / `--list-file` / `--single` / `--output` — the
@@ -706,7 +724,7 @@ export class CrawlerOrchestrator extends EventEmitter {
      *   `inventoryUrls` in-memory; the audit row's `source_file_sha256`
      *   column will be `NULL` and no source list is archived.
      * @returns The orchestrator instance after a successful inventory pass.
-     * @throws {Error} When `inventoryUrls` is empty, the archive is in list mode, or pending URLs from a previous crawl remain unresolved.
+     * @throws {Error} When `inventoryUrls` is empty or the archive is in list mode. Unresolved pending URLs from a previous crawl do NOT throw — see step 3.
      */
     static async inventory(archivePath, inventoryUrls, options, initializedCallback, source = null) {
         if (inventoryUrls.length === 0) {
@@ -795,17 +813,49 @@ export class CrawlerOrchestrator extends EventEmitter {
             });
             const knownCount = existingPageUrls.size + existingResourceUrls.size;
             log('[inventory] %d in-scope, %d already in archive, %d new', inScope.length, knownCount, novelUrls.length);
+            // Split the novel URLs on the exclusion config BEFORE the
+            // HTML/non-HTML classification, so an exclude-matched URL is
+            // recorded as a terminal skipped page instead of being imported
+            // (issue #260). The inputs mirror the scrape phase's fetch-time
+            // gate (`shouldSkipUrl` in `crawler.ts` fed by the constructor's
+            // merge): archived config overlaid with this run's overrides,
+            // and `DEFAULT_EXCLUDED_EXTERNAL_URLS` merged ahead of the
+            // user's prefixes — classification and gate must never disagree
+            // about the same URL. Running this AFTER the known-URL filter is
+            // deliberate: a previously crawled row that newly matches the
+            // exclusion config stays untouched (crawled-wins), matching how
+            // `getExistingPageUrls` shields known rows from re-labelling.
+            // `excludeKeywords` is deliberately absent: it matches rendered
+            // page content, which a URL list does not have — HTML seeds
+            // still get it at render time via the browser verdict.
+            const effectiveConfig = { ...archived, ...cleanObject(options) };
+            const excludes = normalizeToArray(effectiveConfig.excludes);
+            const excludeUrls = [
+                ...DEFAULT_EXCLUDED_EXTERNAL_URLS,
+                ...normalizeToArray(effectiveConfig.excludeUrls),
+            ];
+            const excludedNovelUrls = [];
+            const importableNovelUrls = [];
+            for (const url of novelUrls) {
+                if (shouldSkipUrl({ url, excludes, excludeUrls, options: effectiveConfig })) {
+                    excludedNovelUrls.push(url);
+                }
+                else {
+                    importableNovelUrls.push(url);
+                }
+            }
+            if (excludedNovelUrls.length > 0) {
+                log('[inventory] %d URL(s) recorded as skipped (matched excludes / excludeUrls)', excludedNovelUrls.length);
+            }
             if (novelUrls.length === 0) {
                 // Nothing to do — release the archive cleanly without taking a
                 // backup. The orchestrator returned here is empty; the caller
-                // should only invoke `close` on it.
-                const noopConfig = {
-                    ...archived,
-                    ...cleanObject(options),
-                };
-                const orchestrator = new CrawlerOrchestrator(archive, noopConfig);
+                // should only invoke `close` on it. `effectiveConfig` is the
+                // same archived-plus-overrides merge every other path in this
+                // method sees.
+                const orchestrator = new CrawlerOrchestrator(archive, effectiveConfig);
                 if (initializedCallback) {
-                    await initializedCallback(orchestrator, noopConfig);
+                    await initializedCallback(orchestrator, effectiveConfig);
                 }
                 return orchestrator;
             }
@@ -821,7 +871,7 @@ export class CrawlerOrchestrator extends EventEmitter {
             // clause). This flag steers the catch below.
             let ingestionComplete = false;
             try {
-                // Classify novel URLs by URL-extension heuristic (no I/O).
+                // Classify importable novel URLs by URL-extension heuristic (no I/O).
                 // Source file lists come from `ls` on the doc-root, so the
                 // extension reflects the real file type — a HEAD pre-flight
                 // here would be pure wasted I/O. Edge cases:
@@ -852,7 +902,7 @@ export class CrawlerOrchestrator extends EventEmitter {
                 // null as "not probed" rather than "failed".
                 const rawHtmlSeeds = [];
                 const nonHtmlSeeds = [];
-                for (const url of novelUrls) {
+                for (const url of importableNovelUrls) {
                     if (isLikelyHtmlUrl(url)) {
                         rawHtmlSeeds.push(url);
                     }
@@ -892,7 +942,15 @@ export class CrawlerOrchestrator extends EventEmitter {
                 // these rows up on the next `--resume` via the
                 // `OR p.source != 'crawled'` clause.
                 await archive.insertInventorySeeds(htmlSeeds);
-                log('[inventory] %d HTML seed(s), %d non-HTML resource(s) recorded', htmlSeeds.length, nonHtmlSeeds.length);
+                // Record exclude-matched novel URLs as terminal skipped pages
+                // (`is_skipped=1`, `skip_reason='excluded'`,
+                // `source='inventory-seed'`) — the same end state the normal
+                // crawl's fetch-time gate produces for link-discovered
+                // excluded URLs, so the archive looks identical no matter
+                // how the URL was discovered. Inside the `.bak` window for
+                // the same all-or-nothing reason as the seed inserts above.
+                await archive.insertInventorySkippedPages(excludedNovelUrls);
+                log('[inventory] %d HTML seed(s), %d non-HTML resource(s), %d skipped page(s) recorded', htmlSeeds.length, nonHtmlSeeds.length, excludedNovelUrls.length);
                 // Audit row is written *inside* the `.bak` window: a libsql
                 // hiccup or transient lock on the INSERT aborts the ingestion
                 // and the `.bak` restore wipes the pre-inserted seeds too,
@@ -906,6 +964,7 @@ export class CrawlerOrchestrator extends EventEmitter {
                     htmlSeedsCount: htmlSeeds.length,
                     nonHtmlCount: nonHtmlSeeds.length,
                     outOfScope,
+                    excludeSkipped: excludedNovelUrls.length,
                     sourceFileSha256: source?.sha256 ?? null,
                     invalidSkipped: source?.invalidLineCount ?? null,
                 });
@@ -925,8 +984,7 @@ export class CrawlerOrchestrator extends EventEmitter {
                 // (matches the rest of the orchestrator's public surface —
                 // no inventory bookkeeping leaks out).
                 const baseConfig = {
-                    ...archived,
-                    ...cleanObject(options),
+                    ...effectiveConfig,
                     recursive: true,
                     fromList: false,
                 };
@@ -1252,6 +1310,7 @@ export class CrawlerOrchestrator extends EventEmitter {
             new_pages: aggregates.htmlSeedsCount,
             new_resources: aggregates.nonHtmlCount,
             scope_skipped: aggregates.outOfScope,
+            exclude_skipped: aggregates.excludeSkipped,
             invalid_skipped: aggregates.invalidSkipped,
         });
     }

package/lib/crawler.d.ts

@@ -33,6 +33,7 @@ export { computeBodyHash } from './archive/body-hash/compute-body-hash.js';
 export { decodeStoredBlob } from './archive/decode-html-blob.js';
 export { computeTierAAliasKey } from './archive/url-alias/compute-tier-a-alias-key.js';
 export { computeTierBAliasKey } from './archive/url-alias/compute-tier-b-alias-key.js';
+export { computeShapeKey } from './crawler/dedupe/compute-shape-key.js';
 export { DEFAULT_EXCLUDED_EXTERNAL_URLS, CrawlerOrchestrator, } from './crawler-orchestrator.js';
 export * from './types.js';
 export * from './crawler/types.js';

package/lib/crawler.js

@@ -31,6 +31,7 @@ export { computeBodyHash } from './archive/body-hash/compute-body-hash.js';
 export { decodeStoredBlob } from './archive/decode-html-blob.js';
 export { computeTierAAliasKey } from './archive/url-alias/compute-tier-a-alias-key.js';
 export { computeTierBAliasKey } from './archive/url-alias/compute-tier-b-alias-key.js';
+export { computeShapeKey } from './crawler/dedupe/compute-shape-key.js';
 // Core
 export { DEFAULT_EXCLUDED_EXTERNAL_URLS, CrawlerOrchestrator, } from './crawler-orchestrator.js';
 export * from './types.js';

package/lib/types.d.ts

@@ -18,6 +18,8 @@ export interface InventoryRunAggregates {
     nonHtmlCount: number;
     /** URLs dropped because they fell outside the archived scope. Stored as `scope_skipped`. */
     outOfScope: number;
+    /** Novel in-scope URLs recorded as terminal skipped pages (`is_skipped=1`, `skip_reason='excluded'`) instead of being imported, because they matched the effective `excludes` / `excludeUrls` config — the same inputs the scrape phase's fetch-time `shouldSkipUrl` gate uses. Stored as `exclude_skipped`. */
+    excludeSkipped: number;
     /**
      * SHA-256 hex digest of the source `.txt`, **pre-computed by the caller**
      * (typically the CLI's `inventoryCrawl`). Stored verbatim as

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@nitpicker/crawler",
-	"version": "0.16.0",
+	"version": "0.18.0",
 	"description": "Web crawler engine with headless browser rendering and archive storage",
 	"author": "D-ZERO",
 	"license": "Apache-2.0",
@@ -27,10 +27,10 @@
 		"clean": "tsc --build --clean"
 	},
 	"dependencies": {
-		"@d-zero/beholder": "4.1.0",
-		"@d-zero/dealer": "1.9.4",
+		"@d-zero/beholder": "4.2.2",
+		"@d-zero/dealer": "1.10.4",
 		"@d-zero/fs": "0.2.6",
-		"@d-zero/shared": "0.22.2",
+		"@d-zero/shared": "0.22.5",
 		"ansi-colors": "4.1.3",
 		"debug": "4.4.3",
 		"follow-redirects": "1.16.0",
@@ -39,7 +39,7 @@
 		"libsql": "0.5.29",
 		"puppeteer": "25.3.0",
 		"robots-parser": "3.0.1",
-		"tar": "7.5.20"
+		"tar": "7.5.21"
 	},
 	"devDependencies": {
 		"@types/debug": "4.1.13",
@@ -48,5 +48,5 @@
 		"@types/tar": "7.0.87",
 		"@types/unzipper": "0.10.11"
 	},
-	"gitHead": "bef8b6d48e3ca5167fee643d6aba8644a065df4a"
+	"gitHead": "9be5b756adb5bfaaa3e7fd52fabc93213c7e0b50"
 }