@nitpicker/crawler 0.15.0 → 0.17.0

package/lib/archive/db-ops/dedupe-cap/list-dedupe-cap-shape-keys.js

@@ -0,0 +1,27 @@
+/**
+ * Every distinct `dedupe_cap_events.shape_key` recorded in this archive —
+ * used by `CrawlerOrchestrator` to preload `DedupeCapTracker`'s sticky set
+ * on `--resume` / `--append` / `--retry-failed` / `--inventory`, so a trap
+ * this crawl already paid the cost of discovering once is not re-admitted
+ * in a later session. Fresh (non-resuming) crawls do not call this — there
+ * is no archive history to seed from.
+ *
+ * Unlike `listDnsBurnedHostCandidates`, no additional exclusion logic is
+ * needed: once `DedupeCapTracker` confirms a shape as a trap, it stays
+ * confirmed — there is no equivalent of "the host might have recovered
+ * since".
+ *
+ * Returns `[]` on legacy archives that pre-date the `dedupe_cap_events`
+ * table (self-healed on next writer open, so this is never a permanent
+ * state) or that have recorded no capped shapes.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns Distinct shape keys already confirmed capped.
+ */
+export async function listDedupeCapShapeKeys(knex) {
+    const hasTable = await knex.schema.hasTable('dedupe_cap_events');
+    if (!hasTable) {
+        return [];
+    }
+    const rows = (await knex('dedupe_cap_events').distinct('shape_key'));
+    return rows.map((row) => row.shape_key);
+}

package/lib/archive/db-ops/inventory/record-inventory-run.js

@@ -27,6 +27,7 @@ export async function recordInventoryRun(knex, meta) {
         new_pages: meta.new_pages ?? null,
         new_resources: meta.new_resources ?? null,
         scope_skipped: meta.scope_skipped ?? null,
+        exclude_skipped: meta.exclude_skipped ?? null,
         invalid_skipped: meta.invalid_skipped ?? null,
         notes: meta.notes ?? null,
     })

package/lib/archive/db-ops/lifecycle/init.d.ts

@@ -2,7 +2,8 @@ import type { Knex } from 'knex';
 /**
  * Initializes the database schema if tables do not exist, then runs the
  * remaining lightweight migrations (`info.roots`, `info.mainContentSelector`,
- * `page_meta.main_content_*`, `inventory_runs.invalid_skipped`).
+ * `page_meta.main_content_*`, `inventory_runs.invalid_skipped`,
+ * `inventory_runs.exclude_skipped`).
  *
  * There is deliberately no per-table *table-creation* migration chain here:
  * `assertCompatibleVersion` (called below, before any schema work) rejects
@@ -19,7 +20,8 @@ import type { Knex } from 'knex';
  * needs an explicit `hasColumn`-guarded `ALTER TABLE` here (`migrateInfoRoots`,
  * `migrateMainContentsColumns`, `migratePageMetaBodyHash`,
  * `migratePageMetaConsoleErrorCount`, `migrateContentItemsAliasOfId`,
- * `migrateInventoryRunsInvalidSkipped`) rather than a DDL-string change alone.
+ * `migrateContentItemsDedupeCapEventId`, `migrateInventoryRunsInvalidSkipped`,
+ * `migrateInventoryRunsExcludeSkipped`) rather than a DDL-string change alone.
  *
  * `closeStaleOpenNetworkOutages` is not a schema migration (no columns
  * change) but belongs at this same boot phase for the same reason the

package/lib/archive/db-ops/lifecycle/init.js

@@ -1,8 +1,10 @@
 import { applyConnectionPragmas, initSchema } from '../../init-schema.js';
 import { assertCompatibleVersion } from '../../meta/assert-compatible-version.js';
 import { migrateContentItemsAliasOfId } from '../../migrate-content-items-alias-of-id.js';
+import { migrateContentItemsDedupeCapEventId } from '../../migrate-content-items-dedupe-cap-event-id.js';
 import { migrateInfoMainContentSelector } from '../../migrate-info-main-content-selector.js';
 import { migrateInfoRoots } from '../../migrate-info-roots.js';
+import { migrateInventoryRunsExcludeSkipped } from '../../migrate-inventory-runs-exclude-skipped.js';
 import { migrateInventoryRunsInvalidSkipped } from '../../migrate-inventory-runs-invalid-skipped.js';
 import { migrateMainContentsColumns } from '../../migrate-main-contents-columns.js';
 import { migratePageMetaBodyHash } from '../../migrate-page-meta-body-hash.js';
@@ -11,7 +13,8 @@ import { closeStaleOpenNetworkOutages } from '../outages/close-stale-open-networ
 /**
  * Initializes the database schema if tables do not exist, then runs the
  * remaining lightweight migrations (`info.roots`, `info.mainContentSelector`,
- * `page_meta.main_content_*`, `inventory_runs.invalid_skipped`).
+ * `page_meta.main_content_*`, `inventory_runs.invalid_skipped`,
+ * `inventory_runs.exclude_skipped`).
  *
  * There is deliberately no per-table *table-creation* migration chain here:
  * `assertCompatibleVersion` (called below, before any schema work) rejects
@@ -28,7 +31,8 @@ import { closeStaleOpenNetworkOutages } from '../outages/close-stale-open-networ
  * needs an explicit `hasColumn`-guarded `ALTER TABLE` here (`migrateInfoRoots`,
  * `migrateMainContentsColumns`, `migratePageMetaBodyHash`,
  * `migratePageMetaConsoleErrorCount`, `migrateContentItemsAliasOfId`,
- * `migrateInventoryRunsInvalidSkipped`) rather than a DDL-string change alone.
+ * `migrateContentItemsDedupeCapEventId`, `migrateInventoryRunsInvalidSkipped`,
+ * `migrateInventoryRunsExcludeSkipped`) rather than a DDL-string change alone.
  *
  * `closeStaleOpenNetworkOutages` is not a schema migration (no columns
  * change) but belongs at this same boot phase for the same reason the
@@ -66,6 +70,12 @@ export async function init(knex, readOnly) {
     await migratePageMetaBodyHash(knex);
     await migratePageMetaConsoleErrorCount(knex);
     await migrateContentItemsAliasOfId(knex);
+    // Runs after `initSchema` above, which already created
+    // `dedupe_cap_events` (an adjunct table) unconditionally — so the new
+    // column's `REFERENCES dedupe_cap_events(id)` target always exists by
+    // this point, for both fresh and legacy archives.
+    await migrateContentItemsDedupeCapEventId(knex);
     await migrateInventoryRunsInvalidSkipped(knex);
+    await migrateInventoryRunsExcludeSkipped(knex);
     await closeStaleOpenNetworkOutages(knex);
 }

package/lib/archive/db-ops/pages/write/insert-inventory-content-items.d.ts

@@ -0,0 +1,38 @@
+import type { WriteRefCaches } from '../../_shared/types.js';
+import type { Knex } from 'knex';
+/**
+ * Parameters for {@link insertInventoryContentItems}.
+ */
+export interface InsertInventoryContentItemsParams {
+    /** Knex query builder connected to the archive DB. */
+    readonly knex: Knex;
+    /** The connection's write-side id caches. */
+    readonly caches: WriteRefCaches;
+    /** URL strings already in `withoutHashAndAuth` form. */
+    readonly urls: readonly string[];
+    /** `content_items` column values shared by every inserted row (everything except `url_id`). */
+    readonly row: Readonly<Record<string, number | string>>;
+    /** Calling op's name, used to prefix the unresolved-url_ref error message. */
+    readonly opName: string;
+}
+/**
+ * Shared body of the inventory `content_items` bulk-insert ops
+ * (`insertInventorySeeds` / `insertInventorySkippedPages`): chunked
+ * `url_refs` upsert → id resolution → `content_items` insert-ignore →
+ * write-cache population. The two callers differ only in the row
+ * constants they stamp on every row, so the invariant-heavy plumbing
+ * lives here exactly once:
+ *
+ * - Chunked into 500-URL batches so SQLite's bound-parameter limit
+ *   (`SQLITE_MAX_VARIABLE_NUMBER`) cannot be hit even on a
+ *   tens-of-thousands inventory list.
+ * - Both inserts are `ON CONFLICT ... IGNORE`, so existing rows — in
+ *   particular previously crawled pages — are never overwritten
+ *   (crawled-wins), and within-list duplicates collapse to one row.
+ * - The `urlIds` / `contentItems` write caches are populated from what
+ *   the DB actually holds after the insert (not from the attempted row
+ *   values), keeping later cache-hits consistent with conflict-ignored
+ *   rows.
+ * @param params - See {@link InsertInventoryContentItemsParams}.
+ */
+export declare function insertInventoryContentItems(params: InsertInventoryContentItemsParams): Promise<void>;

package/lib/archive/db-ops/pages/write/insert-inventory-content-items.js

@@ -0,0 +1,59 @@
+import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+import { resolveUrlRefs } from '../../../populate-entity-tables/resolve-url-refs.js';
+import { decomposeUrl } from '../../../populate-ref-tables/decompose-url.js';
+/**
+ * Shared body of the inventory `content_items` bulk-insert ops
+ * (`insertInventorySeeds` / `insertInventorySkippedPages`): chunked
+ * `url_refs` upsert → id resolution → `content_items` insert-ignore →
+ * write-cache population. The two callers differ only in the row
+ * constants they stamp on every row, so the invariant-heavy plumbing
+ * lives here exactly once:
+ *
+ * - Chunked into 500-URL batches so SQLite's bound-parameter limit
+ *   (`SQLITE_MAX_VARIABLE_NUMBER`) cannot be hit even on a
+ *   tens-of-thousands inventory list.
+ * - Both inserts are `ON CONFLICT ... IGNORE`, so existing rows — in
+ *   particular previously crawled pages — are never overwritten
+ *   (crawled-wins), and within-list duplicates collapse to one row.
+ * - The `urlIds` / `contentItems` write caches are populated from what
+ *   the DB actually holds after the insert (not from the attempted row
+ *   values), keeping later cache-hits consistent with conflict-ignored
+ *   rows.
+ * @param params - See {@link InsertInventoryContentItemsParams}.
+ */
+export async function insertInventoryContentItems(params) {
+    const { knex, caches, urls, row, opName } = params;
+    if (urls.length === 0) {
+        return;
+    }
+    await eachSplitted([...urls], 500, async (chunk) => {
+        await knex('url_refs')
+            .insert(chunk.map((url) => ({ url, ...decomposeUrl(url) })))
+            .onConflict('url')
+            .ignore();
+        const urlIds = await resolveUrlRefs(knex, chunk);
+        const rows = chunk.map((url) => {
+            const urlId = urlIds.get(url);
+            if (urlId === undefined) {
+                throw new Error(`${opName}: url_refs.id not resolved for ${url}`);
+            }
+            caches.urlIds.set(url, urlId);
+            return {
+                url_id: urlId,
+                ...row,
+            };
+        });
+        await knex('content_items').insert(rows).onConflict('url_id').ignore();
+        const inserted = (await knex
+            .select('ci.id', 'ci.source', 'ur.url')
+            .from('content_items as ci')
+            .join('url_refs as ur', 'ur.id', 'ci.url_id')
+            .whereIn('ur.url', chunk));
+        for (const insertedRow of inserted) {
+            caches.contentItems.set(insertedRow.url, {
+                id: insertedRow.id,
+                source: insertedRow.source,
+            });
+        }
+    });
+}

package/lib/archive/db-ops/pages/write/insert-inventory-seeds.d.ts

@@ -23,15 +23,14 @@ import type { Knex } from 'knex';
  * behaviour (a seed that turned out to be reachable is not an orphan
  * and should not retain the inventory label).
  *
- * Chunked into 500-URL batches so SQLite's bound-parameter limit
- * (`SQLITE_MAX_VARIABLE_NUMBER`) cannot be hit even on a
- * tens-of-thousands inventory list.
- *
  * Called by `CrawlerOrchestrator.inventory` during the
  * `.bak`-protected ingestion phase, so any failure here aborts the run
- * and restores from backup — the operator reruns from scratch.
+ * and restores from backup — the operator reruns from scratch. The
+ * chunking / conflict-ignore / cache-population plumbing lives in
+ * {@link insertInventoryContentItems}, shared with
+ * `insertInventorySkippedPages`.
  * @param knex - Knex query builder connected to the archive DB.
- * @param caches
+ * @param caches - The connection's write-side id caches.
  * @param urls - URL strings already in `withoutHashAndAuth` form.
  */
 export declare function insertInventorySeeds(knex: Knex, caches: WriteRefCaches, urls: readonly string[]): Promise<void>;

package/lib/archive/db-ops/pages/write/insert-inventory-seeds.js

@@ -1,6 +1,4 @@
-import { eachSplitted } from '../../../../utils/array/each-splitted.js';
-import { resolveUrlRefs } from '../../../populate-entity-tables/resolve-url-refs.js';
-import { decomposeUrl } from '../../../populate-ref-tables/decompose-url.js';
+import { insertInventoryContentItems } from './insert-inventory-content-items.js';
 /**
  * Pre-insert inventory HTML seeds into `content_items` as `scraped = 0`,
  * `source = 'inventory-seed'` placeholders so the URL's existence in the
@@ -24,49 +22,27 @@ import { decomposeUrl } from '../../../populate-ref-tables/decompose-url.js';
  * behaviour (a seed that turned out to be reachable is not an orphan
  * and should not retain the inventory label).
  *
- * Chunked into 500-URL batches so SQLite's bound-parameter limit
- * (`SQLITE_MAX_VARIABLE_NUMBER`) cannot be hit even on a
- * tens-of-thousands inventory list.
- *
  * Called by `CrawlerOrchestrator.inventory` during the
  * `.bak`-protected ingestion phase, so any failure here aborts the run
- * and restores from backup — the operator reruns from scratch.
+ * and restores from backup — the operator reruns from scratch. The
+ * chunking / conflict-ignore / cache-population plumbing lives in
+ * {@link insertInventoryContentItems}, shared with
+ * `insertInventorySkippedPages`.
  * @param knex - Knex query builder connected to the archive DB.
- * @param caches
+ * @param caches - The connection's write-side id caches.
  * @param urls - URL strings already in `withoutHashAndAuth` form.
  */
 export async function insertInventorySeeds(knex, caches, urls) {
-    if (urls.length === 0) {
-        return;
-    }
-    await eachSplitted([...urls], 500, async (chunk) => {
-        await knex('url_refs')
-            .insert(chunk.map((url) => ({ url, ...decomposeUrl(url) })))
-            .onConflict('url')
-            .ignore();
-        const urlIds = await resolveUrlRefs(knex, chunk);
-        const rows = chunk.map((url) => {
-            const urlId = urlIds.get(url);
-            if (urlId === undefined) {
-                throw new Error(`insertInventorySeeds: url_refs.id not resolved for ${url}`);
-            }
-            caches.urlIds.set(url, urlId);
-            return {
-                url_id: urlId,
-                scraped: 0,
-                is_external: 0,
-                is_target: 0,
-                source: 'inventory-seed',
-            };
-        });
-        await knex('content_items').insert(rows).onConflict('url_id').ignore();
-        const inserted = (await knex
-            .select('ci.id', 'ci.source', 'ur.url')
-            .from('content_items as ci')
-            .join('url_refs as ur', 'ur.id', 'ci.url_id')
-            .whereIn('ur.url', chunk));
-        for (const row of inserted) {
-            caches.contentItems.set(row.url, { id: row.id, source: row.source });
-        }
+    await insertInventoryContentItems({
+        knex,
+        caches,
+        urls,
+        row: {
+            scraped: 0,
+            is_external: 0,
+            is_target: 0,
+            source: 'inventory-seed',
+        },
+        opName: 'insertInventorySeeds',
     });
 }

package/lib/archive/db-ops/pages/write/insert-inventory-skipped-pages.d.ts

@@ -0,0 +1,42 @@
+import type { WriteRefCaches } from '../../_shared/types.js';
+import type { Knex } from 'knex';
+/**
+ * Record exclude-matched inventory URLs into `content_items` as
+ * `scraped = 1`, `is_skipped = 1`, `skip_reason = 'excluded'`,
+ * `source = 'inventory-seed'` rows — the same terminal state the normal
+ * crawl's fetch-time `shouldSkipUrl` gate produces via `setSkippedPage`
+ * for link-discovered excluded URLs.
+ *
+ * Why a dedicated write path instead of routing these URLs through the
+ * crawler's gate: non-HTML inventory URLs never enter the crawler at all
+ * (they are recorded straight into `resources`), so the gate cannot see
+ * them, and pre-inserting HTML seeds only to have the dealer skip them
+ * wastes dealer slots for a verdict already known at ingestion time.
+ * Writing the terminal skipped state directly keeps the invariant "the
+ * same URL lands in the same archive state regardless of how it was
+ * discovered (anchor vs inventory list)" for both classifications
+ * (issue #260).
+ *
+ * `scraped = 1` is load-bearing: it keeps these rows out of
+ * `getCrawlingState`'s strict pending set, so `--resume` after an
+ * interrupted inventory pass does not try to fetch operator-excluded
+ * URLs.
+ *
+ * Idempotent: both the `url_refs` and `content_items` inserts are
+ * `ON CONFLICT ... IGNORE`, so an existing row — in particular a
+ * previously crawled page that now matches the exclusion config — is
+ * never downgraded to skipped by this path (crawled-wins). The
+ * orchestrator additionally filters known URLs out before calling this,
+ * so conflicts here are limited to within-list duplicates.
+ *
+ * Called by `CrawlerOrchestrator.inventory` during the `.bak`-protected
+ * ingestion phase, so any failure here aborts the run and restores from
+ * backup — the operator reruns from scratch. The chunking /
+ * conflict-ignore / cache-population plumbing lives in
+ * {@link insertInventoryContentItems}, shared with
+ * `insertInventorySeeds`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param urls - URL strings already in `withoutHashAndAuth` form.
+ */
+export declare function insertInventorySkippedPages(knex: Knex, caches: WriteRefCaches, urls: readonly string[]): Promise<void>;

package/lib/archive/db-ops/pages/write/insert-inventory-skipped-pages.js

@@ -0,0 +1,56 @@
+import { insertInventoryContentItems } from './insert-inventory-content-items.js';
+/**
+ * Record exclude-matched inventory URLs into `content_items` as
+ * `scraped = 1`, `is_skipped = 1`, `skip_reason = 'excluded'`,
+ * `source = 'inventory-seed'` rows — the same terminal state the normal
+ * crawl's fetch-time `shouldSkipUrl` gate produces via `setSkippedPage`
+ * for link-discovered excluded URLs.
+ *
+ * Why a dedicated write path instead of routing these URLs through the
+ * crawler's gate: non-HTML inventory URLs never enter the crawler at all
+ * (they are recorded straight into `resources`), so the gate cannot see
+ * them, and pre-inserting HTML seeds only to have the dealer skip them
+ * wastes dealer slots for a verdict already known at ingestion time.
+ * Writing the terminal skipped state directly keeps the invariant "the
+ * same URL lands in the same archive state regardless of how it was
+ * discovered (anchor vs inventory list)" for both classifications
+ * (issue #260).
+ *
+ * `scraped = 1` is load-bearing: it keeps these rows out of
+ * `getCrawlingState`'s strict pending set, so `--resume` after an
+ * interrupted inventory pass does not try to fetch operator-excluded
+ * URLs.
+ *
+ * Idempotent: both the `url_refs` and `content_items` inserts are
+ * `ON CONFLICT ... IGNORE`, so an existing row — in particular a
+ * previously crawled page that now matches the exclusion config — is
+ * never downgraded to skipped by this path (crawled-wins). The
+ * orchestrator additionally filters known URLs out before calling this,
+ * so conflicts here are limited to within-list duplicates.
+ *
+ * Called by `CrawlerOrchestrator.inventory` during the `.bak`-protected
+ * ingestion phase, so any failure here aborts the run and restores from
+ * backup — the operator reruns from scratch. The chunking /
+ * conflict-ignore / cache-population plumbing lives in
+ * {@link insertInventoryContentItems}, shared with
+ * `insertInventorySeeds`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param urls - URL strings already in `withoutHashAndAuth` form.
+ */
+export async function insertInventorySkippedPages(knex, caches, urls) {
+    await insertInventoryContentItems({
+        knex,
+        caches,
+        urls,
+        row: {
+            scraped: 1,
+            is_external: 0,
+            is_target: 0,
+            is_skipped: 1,
+            skip_reason: 'excluded',
+            source: 'inventory-seed',
+        },
+        opName: 'insertInventorySkippedPages',
+    });
+}

package/lib/archive/migrate-content-items-dedupe-cap-event-id.d.ts

@@ -0,0 +1,41 @@
+import type { Knex } from 'knex';
+/**
+ * Adds the `content_items.dedupe_cap_event_id` column to archives created
+ * before this feature.
+ *
+ * `content_items` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `content_items` predates this change. This
+ * mirrors {@link import('./migrate-content-items-alias-of-id.js').migrateContentItemsAliasOfId}'s
+ * catch-up: a `hasColumn`-guarded `ALTER TABLE` for the one column
+ * `CREATE TABLE IF NOT EXISTS` cannot retrofit.
+ *
+ * Uses a raw `ALTER TABLE` (not the knex schema builder) so the retrofitted
+ * column's `REFERENCES dedupe_cap_events(id) DEFERRABLE INITIALLY DEFERRED`
+ * constraint matches the fresh-archive DDL bit-for-bit.
+ *
+ * Unlike `migrateContentItemsAliasOfId`, this migration never creates an
+ * index for the column — `--dedupe-cap` is opt-in and the number of rows a
+ * cap event ever marks is small (capped shapes × matching URLs), so there is
+ * no measured hot path to justify one. See `createEntityTables`'s DDL
+ * comment for the same reasoning.
+ *
+ * Only adds the column — it does not compute values for existing rows (they
+ * stay `NULL`). That computation runs separately, from
+ * `backfillDedupeCapEventId` during a viewer-read-model build, since it
+ * requires recomputing `computeShapeKey` against every internal page's URL
+ * and matching it against `dedupe_cap_events.shape_key`.
+ *
+ * Idempotent: adding the column is a no-op once it exists. Guards on
+ * `content_items`'s existence defensively, though by the time this runs
+ * (after `initSchema`, itself after `assertCompatibleVersion` rejects
+ * pre-0.13 archives) the table is always present.
+ * @param instance - The Knex query builder instance connected to the database.
+ * @example
+ * ```ts
+ * await migrateContentItemsDedupeCapEventId(knex);
+ * ```
+ */
+export declare function migrateContentItemsDedupeCapEventId(instance: Knex): Promise<void>;

package/lib/archive/migrate-content-items-dedupe-cap-event-id.js

@@ -0,0 +1,51 @@
+/**
+ * Adds the `content_items.dedupe_cap_event_id` column to archives created
+ * before this feature.
+ *
+ * `content_items` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `content_items` predates this change. This
+ * mirrors {@link import('./migrate-content-items-alias-of-id.js').migrateContentItemsAliasOfId}'s
+ * catch-up: a `hasColumn`-guarded `ALTER TABLE` for the one column
+ * `CREATE TABLE IF NOT EXISTS` cannot retrofit.
+ *
+ * Uses a raw `ALTER TABLE` (not the knex schema builder) so the retrofitted
+ * column's `REFERENCES dedupe_cap_events(id) DEFERRABLE INITIALLY DEFERRED`
+ * constraint matches the fresh-archive DDL bit-for-bit.
+ *
+ * Unlike `migrateContentItemsAliasOfId`, this migration never creates an
+ * index for the column — `--dedupe-cap` is opt-in and the number of rows a
+ * cap event ever marks is small (capped shapes × matching URLs), so there is
+ * no measured hot path to justify one. See `createEntityTables`'s DDL
+ * comment for the same reasoning.
+ *
+ * Only adds the column — it does not compute values for existing rows (they
+ * stay `NULL`). That computation runs separately, from
+ * `backfillDedupeCapEventId` during a viewer-read-model build, since it
+ * requires recomputing `computeShapeKey` against every internal page's URL
+ * and matching it against `dedupe_cap_events.shape_key`.
+ *
+ * Idempotent: adding the column is a no-op once it exists. Guards on
+ * `content_items`'s existence defensively, though by the time this runs
+ * (after `initSchema`, itself after `assertCompatibleVersion` rejects
+ * pre-0.13 archives) the table is always present.
+ * @param instance - The Knex query builder instance connected to the database.
+ * @example
+ * ```ts
+ * await migrateContentItemsDedupeCapEventId(knex);
+ * ```
+ */
+export async function migrateContentItemsDedupeCapEventId(instance) {
+    const hasContentItems = await instance.schema.hasTable('content_items');
+    if (!hasContentItems) {
+        return;
+    }
+    const hasColumn = await instance.schema.hasColumn('content_items', 'dedupe_cap_event_id');
+    if (!hasColumn) {
+        await instance.raw('ALTER TABLE content_items ADD COLUMN dedupe_cap_event_id INTEGER REFERENCES dedupe_cap_events(id) DEFERRABLE INITIALLY DEFERRED');
+        // eslint-disable-next-line no-console
+        console.error('[migrate] content_items.dedupe_cap_event_id column added');
+    }
+}

package/lib/archive/migrate-entity-tables.d.ts

@@ -29,6 +29,16 @@ import type { Knex } from 'knex';
  * `scripts/migrate-to-0.13.mjs` orders the two calls statically so this
  * ordering is enforced there, not here.
  *
+ * `content_items` also declares `dedupe_cap_event_id REFERENCES
+ * dedupe_cap_events(id)` — an adjunct table (`createAdjunctTables`), not a
+ * ref table. This function only creates the empty tables (schema-only, see
+ * above), so it does not itself need `dedupe_cap_events` to exist. Callers
+ * that write data into `content_items` afterward do: under `PRAGMA
+ * foreign_keys = ON`, SQLite refuses to even prepare an INSERT/UPDATE
+ * against a table with an unresolvable `REFERENCES` target, so
+ * `scripts/migrate-to-0.13.mjs` creates adjunct tables immediately after
+ * this migration and before any entity-table data write.
+ *
  * **Idempotency**: `createEntityTables` itself uses
  * `CREATE TABLE IF NOT EXISTS` / `CREATE INDEX IF NOT EXISTS` for every
  * statement, so calling it multiple times against any DB state is safe.

package/lib/archive/migrate-entity-tables.js

@@ -29,6 +29,16 @@ import { createEntityTables } from './create-entity-tables.js';
  * `scripts/migrate-to-0.13.mjs` orders the two calls statically so this
  * ordering is enforced there, not here.
  *
+ * `content_items` also declares `dedupe_cap_event_id REFERENCES
+ * dedupe_cap_events(id)` — an adjunct table (`createAdjunctTables`), not a
+ * ref table. This function only creates the empty tables (schema-only, see
+ * above), so it does not itself need `dedupe_cap_events` to exist. Callers
+ * that write data into `content_items` afterward do: under `PRAGMA
+ * foreign_keys = ON`, SQLite refuses to even prepare an INSERT/UPDATE
+ * against a table with an unresolvable `REFERENCES` target, so
+ * `scripts/migrate-to-0.13.mjs` creates adjunct tables immediately after
+ * this migration and before any entity-table data write.
+ *
  * **Idempotency**: `createEntityTables` itself uses
  * `CREATE TABLE IF NOT EXISTS` / `CREATE INDEX IF NOT EXISTS` for every
  * statement, so calling it multiple times against any DB state is safe.

package/lib/archive/migrate-inventory-runs-exclude-skipped.d.ts

@@ -0,0 +1,20 @@
+import type { Knex } from 'knex';
+/**
+ * Adds the `inventory_runs.exclude_skipped` column to archives created
+ * before it existed. `CREATE TABLE IF NOT EXISTS` (used for `inventory_runs`
+ * itself) cannot retrofit a new column onto an already-existing table, so
+ * this lightweight, `hasColumn`-guarded `ALTER TABLE` runs on every
+ * `initSchema` call — idempotent, and self-healing for archives whose
+ * provisioning crashed partway through.
+ *
+ * Pre-migration rows stay `NULL`: those runs predate ingestion-side
+ * exclusion (issue #260), so their excluded URLs were imported as real
+ * pages/resources rather than recorded as skipped — `NULL` means "not
+ * measured", not `0`.
+ *
+ * The column is pure audit output: written once per run and read back
+ * only by `listInventoryRuns` display surfaces, never consumed by any
+ * runtime decision — matching `scope_skipped` / `invalid_skipped`.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function migrateInventoryRunsExcludeSkipped(instance: Knex): Promise<void>;

package/lib/archive/migrate-inventory-runs-exclude-skipped.js

@@ -0,0 +1,33 @@
+/**
+ * Adds the `inventory_runs.exclude_skipped` column to archives created
+ * before it existed. `CREATE TABLE IF NOT EXISTS` (used for `inventory_runs`
+ * itself) cannot retrofit a new column onto an already-existing table, so
+ * this lightweight, `hasColumn`-guarded `ALTER TABLE` runs on every
+ * `initSchema` call — idempotent, and self-healing for archives whose
+ * provisioning crashed partway through.
+ *
+ * Pre-migration rows stay `NULL`: those runs predate ingestion-side
+ * exclusion (issue #260), so their excluded URLs were imported as real
+ * pages/resources rather than recorded as skipped — `NULL` means "not
+ * measured", not `0`.
+ *
+ * The column is pure audit output: written once per run and read back
+ * only by `listInventoryRuns` display surfaces, never consumed by any
+ * runtime decision — matching `scope_skipped` / `invalid_skipped`.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function migrateInventoryRunsExcludeSkipped(instance) {
+    const hasTable = await instance.schema.hasTable('inventory_runs');
+    if (!hasTable) {
+        return;
+    }
+    const hasColumn = await instance.schema.hasColumn('inventory_runs', 'exclude_skipped');
+    if (hasColumn) {
+        return;
+    }
+    await instance.schema.table('inventory_runs', (t) => {
+        t.integer('exclude_skipped');
+    });
+    // eslint-disable-next-line no-console
+    console.error('[migrate] inventory_runs.exclude_skipped column added');
+}

package/lib/archive/populate-entity-tables/test-utils/setup-entities-db.d.ts

@@ -8,6 +8,13 @@ import knex from 'knex';
  *   actually read.
  * - The 0.13 ref / header tables (via {@link createRefTables}).
  * - The 0.13 entity tables (via {@link createEntityTables}).
+ * - The 0.13 adjunct tables (via {@link createAdjunctTables}) — required
+ *   because `content_items.dedupe_cap_event_id REFERENCES
+ *   dedupe_cap_events(id)`; under `PRAGMA foreign_keys = ON` (enabled
+ *   below), inserting into `content_items` fails with `no such table:
+ *   dedupe_cap_events` if the adjunct tables were skipped. `initSchema`
+ *   always calls both create functions together, so this mirrors a real
+ *   archive's actual schema rather than an artificially incomplete one.
  *
  * Every 0.13 populate spec calls this to obtain a fresh DB. The
  * caller is responsible for `db.destroy()` (spec `afterEach`).

package/lib/archive/populate-entity-tables/test-utils/setup-entities-db.js

@@ -1,4 +1,5 @@
 import knex from 'knex';
+import { createAdjunctTables } from '../../create-adjunct-tables.js';
 import { createEntityTables } from '../../create-entity-tables.js';
 import { createRefTables } from '../../create-ref-tables.js';
 import { LibsqlDialect } from '../../libsql-dialect.js';
@@ -11,6 +12,13 @@ import { LibsqlDialect } from '../../libsql-dialect.js';
  *   actually read.
  * - The 0.13 ref / header tables (via {@link createRefTables}).
  * - The 0.13 entity tables (via {@link createEntityTables}).
+ * - The 0.13 adjunct tables (via {@link createAdjunctTables}) — required
+ *   because `content_items.dedupe_cap_event_id REFERENCES
+ *   dedupe_cap_events(id)`; under `PRAGMA foreign_keys = ON` (enabled
+ *   below), inserting into `content_items` fails with `no such table:
+ *   dedupe_cap_events` if the adjunct tables were skipped. `initSchema`
+ *   always calls both create functions together, so this mirrors a real
+ *   archive's actual schema rather than an artificially incomplete one.
  *
  * Every 0.13 populate spec calls this to obtain a fresh DB. The
  * caller is responsible for `db.destroy()` (spec `afterEach`).
@@ -174,5 +182,6 @@ export async function setupMigrationDb() {
 	`);
     await createRefTables(db);
     await createEntityTables(db);
+    await createAdjunctTables(db);
     return db;
 }