@nitpicker/archive 0.0.0-stage → 0.25.0

package/lib/db-ops/pages/reset/reset-pages-by-urls.js

@@ -0,0 +1,151 @@
+import { dbLog } from '../../../debug.js';
+import { eachSplitted } from '../../../utils/array/each-splitted.js';
+import { clearPageDerivedRows } from './clear-page-derived-rows.js';
+/**
+ * Reset pages matching an operator-supplied URL list back to pending, so a
+ * follow-up crawl re-fetches them from scratch — the un-scrape half of
+ * `crawl --recrawl`.
+ *
+ * Unlike {@link resetFailedPages}, this function does not filter by prior
+ * failure kind: a URL named explicitly by the operator is reset regardless
+ * of what its last observation was, including a definitive `4xx`/`5xx` — the
+ * whole point of `--recrawl` is to catch a page that used to 404 and is now
+ * live. The only requirement on the raw candidate scan is `scraped = 1` (a
+ * page never scraped has nothing to reset; it is the caller's "novel URL"
+ * path instead).
+ *
+ * Three conservative guards then apply — matching content follows the same
+ * union used to skip `resetFailedPages`'s permanent-failure filter had none:
+ * a matched row is excluded (not reset) rather than silently promoted, and
+ * the exclusion is reported back to the caller by category:
+ *
+ * - **Redirect source** (`redirect_dest_id` is set): resetting it would
+ *   discard a real, previously-observed 3xx. If the operator wants the
+ *   redirect re-verified, that is `--retry-failed`'s job (which does NOT
+ *   exclude redirect sources from a permanent-failure reset — a different
+ *   operation with a different default).
+ * - **Intentionally skipped** (`is_skipped = 1`): the row was excluded by
+ *   `excludes`/`excludeUrls` at ingestion or fetch time. `--recrawl` does not
+ *   second-guess that configuration; re-running the crawl with a changed
+ *   exclude list is the supported path to un-skip a page.
+ * - **External** (`is_external = 1`): bringing a scope-external URL back
+ *   into scope is `--append`'s job (`repromoteExternalPages`), which also
+ *   updates the scope map. `--recrawl` only re-fetches in-scope pages.
+ *
+ * Deliberately does NOT exclude candidates whose URL shape already has a
+ * confirmed same-cluster trap recorded (`dedupe_cap_events.shape_key`) — the
+ * exclusion {@link resetFailedPages} applies for exactly that reason (see its
+ * own JSDoc). The two functions differ on what "the operator asked for this"
+ * means: `resetFailedPages` widens an automatic, unattended SQL scan
+ * (`--retry-failed`), so a known trap is safe to silently skip re-admitting.
+ * `resetPagesByUrls` resets URLs the operator named one-by-one in a file —
+ * the same reasoning that already exempts explicit URLs from the
+ * failure-kind filter above applies here too: an operator who names a
+ * specific trap-shaped URL is not "re-discovering" it by accident, and
+ * silently refusing to reset a row they explicitly listed would be a more
+ * surprising failure mode than honouring the request.
+ *
+ * A URL that matches no `content_items` row at all (not yet known to the
+ * archive) is silently absent from every array on the result — the caller
+ * treats it as a novel URL, the same "not yet known" path `--inventory`
+ * takes. A URL matching a row with `scraped = 0` is likewise absent from
+ * every array: it is already pending, so there is nothing to reset.
+ *
+ * On a match, `content_items` is UPDATEd exactly like
+ * {@link resetFailedPages} (`scraped`/`status`/`status_text`/
+ * `content_type_id`/`content_length`/`header_set_id` cleared,
+ * `first_crawled_at`/`last_crawled_at` preserved), and every derived row is
+ * cleared via {@link clearPageDerivedRows} plus `page_errors` and
+ * the legacy `analysis_violations` table when an archive still carries it.
+ * Current archives no longer create that table, but an archive written by
+ * an earlier version keeps it (FK to `content_items(id)`). The reset only
+ * UPDATEs `content_items`, so no FK violation is reachable here; the stale
+ * violation rows of a re-fetched page are removed because they describe
+ * content that is about to be replaced. The delete is guarded by `hasTable`.
+ * `analysis_text_refs` is a content-hash dictionary shared across pages and
+ * is not touched; an orphaned entry is a harmless, unreferenced row, the
+ * same trade-off already made for `page_html_blobs`.
+ *
+ * SELECT and UPDATE/DELETE statements are chunked to stay below SQLite's
+ * `SQLITE_LIMIT_VARIABLE_NUMBER`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param urls - URL strings to match against `content_items`, already in
+ *   `withoutHashAndAuth` form (mirrors every other URL-list comparison in
+ *   this package, e.g. `getExistingPageUrls`).
+ * @param onProgress - Called after each chunk's DELETE/UPDATE statements
+ *   complete, with the pages processed so far and the total to reset. Omit
+ *   for no reporting (the default; e.g. tests).
+ * @returns The reset URLs, plus the excluded URLs grouped by exclusion
+ *   reason — see {@link ResetPagesByUrlsResult}.
+ */
+export async function resetPagesByUrls(knex, urls, onProgress) {
+    const empty = {
+        resetUrls: [],
+        excludedRedirects: [],
+        excludedSkipped: [],
+        excludedExternal: [],
+    };
+    if (urls.length === 0) {
+        return empty;
+    }
+    const candidates = [];
+    await eachSplitted([...urls], 500, async (chunk) => {
+        const rows = await knex('content_items')
+            .join('url_refs', 'url_refs.id', 'content_items.url_id')
+            .select('content_items.id as id', 'url_refs.url as url', 'content_items.redirect_dest_id as redirect_dest_id', 'content_items.is_skipped as is_skipped', 'content_items.is_external as is_external')
+            .where('content_items.scraped', 1)
+            .whereIn('url_refs.url', chunk);
+        candidates.push(...rows);
+    });
+    if (candidates.length === 0) {
+        return empty;
+    }
+    const resettable = [];
+    const excludedRedirects = [];
+    const excludedSkipped = [];
+    const excludedExternal = [];
+    for (const row of candidates) {
+        if (row.redirect_dest_id != null) {
+            excludedRedirects.push(row.url);
+            continue;
+        }
+        if (row.is_skipped) {
+            excludedSkipped.push(row.url);
+            continue;
+        }
+        if (row.is_external) {
+            excludedExternal.push(row.url);
+            continue;
+        }
+        resettable.push(row);
+    }
+    if (resettable.length === 0) {
+        return { resetUrls: [], excludedRedirects, excludedSkipped, excludedExternal };
+    }
+    const ids = resettable.map((row) => row.id);
+    const resetUrls = resettable.map((row) => row.url);
+    const hasLegacyViolations = await knex.schema.hasTable('analysis_violations');
+    const chunkSize = 500;
+    for (let i = 0; i < ids.length; i += chunkSize) {
+        const chunk = ids.slice(i, i + chunkSize);
+        await knex('content_items').whereIn('id', chunk).update({
+            scraped: 0,
+            status: null,
+            status_text: null,
+            content_type_id: null,
+            content_length: null,
+            header_set_id: null,
+            // `first_crawled_at` / `last_crawled_at` are deliberately left
+            // untouched, matching `resetFailedPages` — the last-success
+            // timestamp records survive the demotion.
+        });
+        await knex('page_errors').whereIn('pageId', chunk).delete();
+        if (hasLegacyViolations) {
+            await knex('analysis_violations').whereIn('page_id', chunk).delete();
+        }
+        await clearPageDerivedRows(knex, chunk);
+        onProgress?.(Math.min(i + chunkSize, ids.length), ids.length);
+    }
+    dbLog('Reset %d page(s) matched by URL list back to pending', resetUrls.length);
+    return { resetUrls, excludedRedirects, excludedSkipped, excludedExternal };
+}

package/lib/db-ops/pages/reset/retryable-image-scan-codes.d.ts

@@ -0,0 +1,24 @@
+/**
+ * `page_meta.image_scan_desktop` / `image_scan_mobile` codes that qualify a
+ * page for `--retry-failed`, mirroring `@d-zero/beholder`'s `IMAGE_SCAN_CODE`
+ * (kept as a plain numeric literal list here rather than importing that
+ * constant, since the archive's `page_meta` columns persist across
+ * a `@d-zero/beholder` version bump and must keep meaning the same numbers
+ * even if a future crawler build pins an older beholder).
+ *
+ * - `2` (`nav-unsettled`) and `3` (`frame-lost`) are transient failures the
+ *   scan itself could not recover from — a retry is likely to succeed.
+ * - `255` (`unknown`) is retried too, erring on the side of investigation,
+ *   consistent with {@link import('../../../error-kind/classify-error-kind.js').classifyErrorKind}'s
+ *   "unknown" kind never being treated as permanent.
+ *
+ * Deliberately excluded:
+ * - `0` (`ok`) and `1` (`degraded`) — the scan produced data.
+ * - `4` (`scroll-height-exceeded`) — a deterministic, page-shape-driven
+ *   outcome that a retry cannot change.
+ * @example
+ * ```ts
+ * qb.whereIn('page_meta.image_scan_desktop', RETRYABLE_IMAGE_SCAN_CODES);
+ * ```
+ */
+export declare const RETRYABLE_IMAGE_SCAN_CODES: readonly [2, 3, 255];

package/lib/db-ops/pages/reset/retryable-image-scan-codes.js

@@ -0,0 +1,24 @@
+/**
+ * `page_meta.image_scan_desktop` / `image_scan_mobile` codes that qualify a
+ * page for `--retry-failed`, mirroring `@d-zero/beholder`'s `IMAGE_SCAN_CODE`
+ * (kept as a plain numeric literal list here rather than importing that
+ * constant, since the archive's `page_meta` columns persist across
+ * a `@d-zero/beholder` version bump and must keep meaning the same numbers
+ * even if a future crawler build pins an older beholder).
+ *
+ * - `2` (`nav-unsettled`) and `3` (`frame-lost`) are transient failures the
+ *   scan itself could not recover from — a retry is likely to succeed.
+ * - `255` (`unknown`) is retried too, erring on the side of investigation,
+ *   consistent with {@link import('../../../error-kind/classify-error-kind.js').classifyErrorKind}'s
+ *   "unknown" kind never being treated as permanent.
+ *
+ * Deliberately excluded:
+ * - `0` (`ok`) and `1` (`degraded`) — the scan produced data.
+ * - `4` (`scroll-height-exceeded`) — a deterministic, page-shape-driven
+ *   outcome that a retry cannot change.
+ * @example
+ * ```ts
+ * qb.whereIn('page_meta.image_scan_desktop', RETRYABLE_IMAGE_SCAN_CODES);
+ * ```
+ */
+export const RETRYABLE_IMAGE_SCAN_CODES = [2, 3, 255];

package/lib/db-ops/pages/write/insert-audios.d.ts

@@ -0,0 +1,14 @@
+import type { MainContentsData } from '../../../utils/types/types.js';
+import type { Knex } from 'knex';
+/**
+ * Replaces the page's `page_main_content_audios` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export declare function insertAudios(pageId: number, mainContents: MainContentsData, trx: Knex.Transaction): Promise<void>;

package/lib/db-ops/pages/write/insert-audios.js

@@ -0,0 +1,25 @@
+import { eachSplitted } from '../../../utils/array/each-splitted.js';
+/**
+ * Replaces the page's `page_main_content_audios` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export async function insertAudios(pageId, mainContents, trx) {
+    if (mainContents.audios.length === 0)
+        return;
+    const rows = mainContents.audios.map((audio, order) => ({
+        pageId,
+        order,
+        src: audio.src,
+    }));
+    await trx('page_main_content_audios').where('pageId', pageId).delete();
+    await eachSplitted(rows, 100, async (chunk) => {
+        await trx('page_main_content_audios').insert(chunk);
+    });
+}

package/lib/db-ops/pages/write/insert-buttons.d.ts

@@ -0,0 +1,14 @@
+import type { MainContentsData } from '../../../utils/types/types.js';
+import type { Knex } from 'knex';
+/**
+ * Replaces the page's `page_main_content_buttons` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export declare function insertButtons(pageId: number, mainContents: MainContentsData, trx: Knex.Transaction): Promise<void>;

package/lib/db-ops/pages/write/insert-buttons.js

@@ -0,0 +1,29 @@
+import { eachSplitted } from '../../../utils/array/each-splitted.js';
+/**
+ * Replaces the page's `page_main_content_buttons` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export async function insertButtons(pageId, mainContents, trx) {
+    if (mainContents.buttons.length === 0)
+        return;
+    const rows = mainContents.buttons.map((button, order) => ({
+        pageId,
+        order,
+        nodeName: button.nodeName,
+        role: button.role,
+        type: button.type,
+        text: button.text,
+        disabled: button.disabled,
+    }));
+    await trx('page_main_content_buttons').where('pageId', pageId).delete();
+    await eachSplitted(rows, 100, async (chunk) => {
+        await trx('page_main_content_buttons').insert(chunk);
+    });
+}

package/lib/db-ops/pages/write/insert-canvases.d.ts

@@ -0,0 +1,14 @@
+import type { MainContentsData } from '../../../utils/types/types.js';
+import type { Knex } from 'knex';
+/**
+ * Replaces the page's `page_main_content_canvases` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export declare function insertCanvases(pageId: number, mainContents: MainContentsData, trx: Knex.Transaction): Promise<void>;

package/lib/db-ops/pages/write/insert-canvases.js

@@ -0,0 +1,26 @@
+import { eachSplitted } from '../../../utils/array/each-splitted.js';
+/**
+ * Replaces the page's `page_main_content_canvases` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export async function insertCanvases(pageId, mainContents, trx) {
+    if (mainContents.canvases.length === 0)
+        return;
+    const rows = mainContents.canvases.map((canvas, order) => ({
+        pageId,
+        order,
+        width: canvas.width,
+        height: canvas.height,
+    }));
+    await trx('page_main_content_canvases').where('pageId', pageId).delete();
+    await eachSplitted(rows, 100, async (chunk) => {
+        await trx('page_main_content_canvases').insert(chunk);
+    });
+}

package/lib/db-ops/pages/write/insert-custom-elements.d.ts

@@ -0,0 +1,27 @@
+import type { MainContentCustomElementCandidate } from '../../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Replaces the page's `page_main_content_custom_elements` rows with the
+ * freshly captured set. Called inside `updatePage`'s transaction, gated on
+ * `page.mainContentCustomElements !== undefined` by the caller — the caller
+ * must NOT call this with a defaulted `[]` when the capture itself failed
+ * (`undefined`), since this function cannot tell "capture found nothing"
+ * apart from "capture didn't run" once it receives a plain array.
+ *
+ * Unlike `insertButtons` / `insertHeadings` etc. (whose empty-array case is
+ * genuinely ambiguous — beholder's `MainContentsData` gives no way to tell
+ * "successfully found zero" from "degraded scrape" — this function always
+ * deletes existing rows once called, including for an empty array: because
+ * the caller already resolved that ambiguity via the `!== undefined` gate,
+ * an empty array here unambiguously means "capture succeeded, found none,"
+ * so stale rows from a previous crawl must be cleared.
+ *
+ * Unlike `insertButtons`, the input is nitpicker's own
+ * `MainContentCustomElementCandidate[]` (from `capture-custom-elements.ts`),
+ * not a slice of beholder's `MainContentsData` — there is no such category
+ * in `MainContentsData`.
+ * @param pageId - The owning `content_items.id`.
+ * @param customElements - The captured custom elements, in document order.
+ * @param trx - The active transaction.
+ */
+export declare function insertCustomElements(pageId: number, customElements: readonly MainContentCustomElementCandidate[], trx: Knex.Transaction): Promise<void>;

package/lib/db-ops/pages/write/insert-custom-elements.js

@@ -0,0 +1,40 @@
+import { eachSplitted } from '../../../utils/array/each-splitted.js';
+/**
+ * Replaces the page's `page_main_content_custom_elements` rows with the
+ * freshly captured set. Called inside `updatePage`'s transaction, gated on
+ * `page.mainContentCustomElements !== undefined` by the caller — the caller
+ * must NOT call this with a defaulted `[]` when the capture itself failed
+ * (`undefined`), since this function cannot tell "capture found nothing"
+ * apart from "capture didn't run" once it receives a plain array.
+ *
+ * Unlike `insertButtons` / `insertHeadings` etc. (whose empty-array case is
+ * genuinely ambiguous — beholder's `MainContentsData` gives no way to tell
+ * "successfully found zero" from "degraded scrape" — this function always
+ * deletes existing rows once called, including for an empty array: because
+ * the caller already resolved that ambiguity via the `!== undefined` gate,
+ * an empty array here unambiguously means "capture succeeded, found none,"
+ * so stale rows from a previous crawl must be cleared.
+ *
+ * Unlike `insertButtons`, the input is nitpicker's own
+ * `MainContentCustomElementCandidate[]` (from `capture-custom-elements.ts`),
+ * not a slice of beholder's `MainContentsData` — there is no such category
+ * in `MainContentsData`.
+ * @param pageId - The owning `content_items.id`.
+ * @param customElements - The captured custom elements, in document order.
+ * @param trx - The active transaction.
+ */
+export async function insertCustomElements(pageId, customElements, trx) {
+    await trx('page_main_content_custom_elements').where('pageId', pageId).delete();
+    if (customElements.length === 0)
+        return;
+    const rows = customElements.map((el, order) => ({
+        pageId,
+        order,
+        nodeName: el.nodeName,
+        elementId: el.elementId,
+        classList: JSON.stringify(el.classList),
+    }));
+    await eachSplitted(rows, 100, async (chunk) => {
+        await trx('page_main_content_custom_elements').insert(chunk);
+    });
+}

package/lib/db-ops/pages/write/insert-headings.d.ts

@@ -0,0 +1,15 @@
+import type { MainContentsData } from '../../../utils/types/types.js';
+import type { Knex } from 'knex';
+/**
+ * Replaces the page's `page_main_content_headings` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null` (non-HTML / external / metadata-only scrapes have
+ * no main-content data to write).
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export declare function insertHeadings(pageId: number, mainContents: MainContentsData, trx: Knex.Transaction): Promise<void>;

package/lib/db-ops/pages/write/insert-headings.js

@@ -0,0 +1,27 @@
+import { eachSplitted } from '../../../utils/array/each-splitted.js';
+/**
+ * Replaces the page's `page_main_content_headings` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null` (non-HTML / external / metadata-only scrapes have
+ * no main-content data to write).
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export async function insertHeadings(pageId, mainContents, trx) {
+    if (mainContents.headings.length === 0)
+        return;
+    const rows = mainContents.headings.map((heading, order) => ({
+        pageId,
+        order,
+        text: heading.text,
+        level: heading.level,
+    }));
+    await trx('page_main_content_headings').where('pageId', pageId).delete();
+    await eachSplitted(rows, 100, async (chunk) => {
+        await trx('page_main_content_headings').insert(chunk);
+    });
+}

package/lib/db-ops/pages/write/insert-iframes.d.ts

@@ -0,0 +1,14 @@
+import type { MainContentsData } from '../../../utils/types/types.js';
+import type { Knex } from 'knex';
+/**
+ * Replaces the page's `page_main_content_iframes` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export declare function insertIframes(pageId: number, mainContents: MainContentsData, trx: Knex.Transaction): Promise<void>;

package/lib/db-ops/pages/write/insert-iframes.js

@@ -0,0 +1,28 @@
+import { eachSplitted } from '../../../utils/array/each-splitted.js';
+/**
+ * Replaces the page's `page_main_content_iframes` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export async function insertIframes(pageId, mainContents, trx) {
+    if (mainContents.iframes.length === 0)
+        return;
+    const rows = mainContents.iframes.map((iframe, order) => ({
+        pageId,
+        order,
+        src: iframe.src,
+        title: iframe.title,
+        width: iframe.width,
+        height: iframe.height,
+    }));
+    await trx('page_main_content_iframes').where('pageId', pageId).delete();
+    await eachSplitted(rows, 100, async (chunk) => {
+        await trx('page_main_content_iframes').insert(chunk);
+    });
+}

package/lib/db-ops/pages/write/insert-inventory-content-items.d.ts

@@ -0,0 +1,38 @@
+import type { WriteRefCaches } from '../../_shared/types.js';
+import type { Knex } from 'knex';
+/**
+ * Parameters for {@link insertInventoryContentItems}.
+ */
+export interface InsertInventoryContentItemsParams {
+    /** Knex query builder connected to the archive DB. */
+    readonly knex: Knex;
+    /** The connection's write-side id caches. */
+    readonly caches: WriteRefCaches;
+    /** URL strings already in `withoutHashAndAuth` form. */
+    readonly urls: readonly string[];
+    /** `content_items` column values shared by every inserted row (everything except `url_id`). */
+    readonly row: Readonly<Record<string, number | string>>;
+    /** Calling op's name, used to prefix the unresolved-url_ref error message. */
+    readonly opName: string;
+}
+/**
+ * Shared body of the inventory `content_items` bulk-insert ops
+ * (`insertInventorySeeds` / `insertInventorySkippedPages`): chunked
+ * `url_refs` upsert → id resolution → `content_items` insert-ignore →
+ * write-cache population. The two callers differ only in the row
+ * constants they stamp on every row, so the invariant-heavy plumbing
+ * lives here exactly once:
+ *
+ * - Chunked into 500-URL batches so SQLite's bound-parameter limit
+ *   (`SQLITE_MAX_VARIABLE_NUMBER`) cannot be hit even on a
+ *   tens-of-thousands inventory list.
+ * - Both inserts are `ON CONFLICT ... IGNORE`, so existing rows — in
+ *   particular previously crawled pages — are never overwritten
+ *   (crawled-wins), and within-list duplicates collapse to one row.
+ * - The `urlIds` / `contentItems` write caches are populated from what
+ *   the DB actually holds after the insert (not from the attempted row
+ *   values), keeping later cache-hits consistent with conflict-ignored
+ *   rows.
+ * @param params - See {@link InsertInventoryContentItemsParams}.
+ */
+export declare function insertInventoryContentItems(params: InsertInventoryContentItemsParams): Promise<void>;

package/lib/db-ops/pages/write/insert-inventory-content-items.js

@@ -0,0 +1,60 @@
+import { resolveUrlRefs } from '../../../populate-entity-tables/resolve-url-refs.js';
+import { decomposeUrl } from '../../../populate-ref-tables/decompose-url.js';
+import { eachSplitted } from '../../../utils/array/each-splitted.js';
+/**
+ * Shared body of the inventory `content_items` bulk-insert ops
+ * (`insertInventorySeeds` / `insertInventorySkippedPages`): chunked
+ * `url_refs` upsert → id resolution → `content_items` insert-ignore →
+ * write-cache population. The two callers differ only in the row
+ * constants they stamp on every row, so the invariant-heavy plumbing
+ * lives here exactly once:
+ *
+ * - Chunked into 500-URL batches so SQLite's bound-parameter limit
+ *   (`SQLITE_MAX_VARIABLE_NUMBER`) cannot be hit even on a
+ *   tens-of-thousands inventory list.
+ * - Both inserts are `ON CONFLICT ... IGNORE`, so existing rows — in
+ *   particular previously crawled pages — are never overwritten
+ *   (crawled-wins), and within-list duplicates collapse to one row.
+ * - The `urlIds` / `contentItems` write caches are populated from what
+ *   the DB actually holds after the insert (not from the attempted row
+ *   values), keeping later cache-hits consistent with conflict-ignored
+ *   rows.
+ * @param params - See {@link InsertInventoryContentItemsParams}.
+ */
+export async function insertInventoryContentItems(params) {
+    const { knex, caches, urls, row, opName } = params;
+    if (urls.length === 0) {
+        return;
+    }
+    await eachSplitted([...urls], 500, async (chunk) => {
+        await knex('url_refs')
+            .insert(chunk.map((url) => ({ url, ...decomposeUrl(url) })))
+            .onConflict('url')
+            .ignore();
+        const urlIds = await resolveUrlRefs(knex, chunk);
+        const rows = chunk.map((url) => {
+            const urlId = urlIds.get(url);
+            if (urlId === undefined) {
+                throw new Error(`${opName}: url_refs.id not resolved for ${url}`);
+            }
+            caches.urlIds.set(url, urlId);
+            return {
+                url_id: urlId,
+                ...row,
+            };
+        });
+        await knex('content_items').insert(rows).onConflict('url_id').ignore();
+        const inserted = (await knex
+            .select('ci.id', 'ci.source', 'ci.is_metadata_only', 'ur.url')
+            .from('content_items as ci')
+            .join('url_refs as ur', 'ur.id', 'ci.url_id')
+            .whereIn('ur.url', chunk));
+        for (const insertedRow of inserted) {
+            caches.contentItems.set(insertedRow.url, {
+                id: insertedRow.id,
+                source: insertedRow.source,
+                isMetadataOnly: insertedRow.is_metadata_only,
+            });
+        }
+    });
+}

package/lib/db-ops/pages/write/insert-inventory-seeds.d.ts

@@ -0,0 +1,36 @@
+import type { WriteRefCaches } from '../../_shared/types.js';
+import type { Knex } from 'knex';
+/**
+ * Pre-insert inventory HTML seeds into `content_items` as `scraped = 0`,
+ * `source = 'inventory-seed'` placeholders so the URL's existence in the
+ * archive is **durable before the scrape phase starts**.
+ *
+ * Why this is the linchpin of `--inventory` Ctrl+C tolerance: without
+ * pre-insertion, HTML seeds live only in the Crawler's in-memory
+ * `LinkList` until the dealer eventually calls `setPage`. A Ctrl+C /
+ * crash before that point loses the seed without trace, and `--resume`
+ * cannot recover it because `getCrawlingState`'s strict pending set
+ * requires a `content_items` row. Pre-inserting fills exactly that gap:
+ * the strict pending set picks these rows up via its
+ * `OR source != 'crawled'` clause, so `--resume` after an interrupted
+ * inventory pass picks every seed back up. See `getCrawlingState` for
+ * the strict-set rationale.
+ *
+ * Idempotent: both the `url_refs` and `content_items` inserts are
+ * `ON CONFLICT ... IGNORE`, keeping existing rows intact. The
+ * `resolveContentItemId` crawled-wins downgrade still fires later when a
+ * crawled-lineage anchor reaches one of these seeds — that's the right
+ * behaviour (a seed that turned out to be reachable is not an orphan
+ * and should not retain the inventory label).
+ *
+ * Called by `CrawlerOrchestrator.inventory` during the
+ * `.bak`-protected ingestion phase, so any failure here aborts the run
+ * and restores from backup — the operator reruns from scratch. The
+ * chunking / conflict-ignore / cache-population plumbing lives in
+ * {@link insertInventoryContentItems}, shared with
+ * `insertInventorySkippedPages`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param urls - URL strings already in `withoutHashAndAuth` form.
+ */
+export declare function insertInventorySeeds(knex: Knex, caches: WriteRefCaches, urls: readonly string[]): Promise<void>;