@nitpicker/crawler 0.11.0 → 0.13.0

package/lib/archive/db-ops/pages/read/get-pages.d.ts

@@ -0,0 +1,11 @@
+import type { DB_Page, PageFilter } from '../../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves pages from the database with optional filtering, pagination via offset and limit.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param filter - An optional {@link PageFilter} to narrow results by content type and origin.
+ * @param offset - The number of rows to skip. Defaults to `0`.
+ * @param limit - The maximum number of rows to return. Defaults to `100000`.
+ * @returns An array of reconstructed {@link DB_Page} rows.
+ */
+export declare function getPages(knex: Knex, filter?: PageFilter, offset?: number, limit?: number): Promise<DB_Page[]>;

package/lib/archive/db-ops/pages/read/get-pages.js

@@ -0,0 +1,51 @@
+import { buildPageQuery } from './build-page-query.js';
+import { reconstructPageRows } from './reconstruct-page-rows.js';
+/**
+ * Retrieves pages from the database with optional filtering, pagination via offset and limit.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param filter - An optional {@link PageFilter} to narrow results by content type and origin.
+ * @param offset - The number of rows to skip. Defaults to `0`.
+ * @param limit - The maximum number of rows to return. Defaults to `100000`.
+ * @returns An array of reconstructed {@link DB_Page} rows.
+ */
+export async function getPages(knex, filter, offset = 0, limit = 100_000) {
+    const q = buildPageQuery(knex);
+    switch (filter) {
+        case 'page': {
+            q.where('ctr.raw', 'text/html').andWhere('ci.is_target', 1);
+            break;
+        }
+        case 'page-included-no-target': {
+            q.where('ctr.raw', 'text/html');
+            break;
+        }
+        case 'external-page': {
+            q.where('ctr.raw', 'text/html').andWhere('ci.is_external', 1);
+            break;
+        }
+        case 'internal-page': {
+            q.where('ctr.raw', 'text/html').andWhere('ci.is_external', 0);
+            break;
+        }
+        case 'no-page': {
+            q.where((qb) => {
+                qb.whereNull('ctr.raw').orWhereNot('ctr.raw', 'text/html');
+            });
+            break;
+        }
+        case 'external-no-page': {
+            q.where((qb) => {
+                qb.whereNull('ctr.raw').orWhereNot('ctr.raw', 'text/html');
+            }).andWhere('ci.is_external', 1);
+            break;
+        }
+        case 'internal-no-page': {
+            q.where((qb) => {
+                qb.whereNull('ctr.raw').orWhereNot('ctr.raw', 'text/html');
+            }).andWhere('ci.is_external', 0);
+            break;
+        }
+    }
+    const rows = await q.limit(limit).offset(offset);
+    return reconstructPageRows(knex, rows);
+}

package/lib/archive/db-ops/pages/read/get-scraped-html-page-count.d.ts

@@ -0,0 +1,18 @@
+import type { Knex } from 'knex';
+/**
+ * Counts pages that were scraped as crawl targets (full HTML render).
+ *
+ * Used by the crawler to seed its `pagesScraped` counter on resume so the
+ * progress display reflects all browser-rendered HTML pages across sessions,
+ * not just the current one.
+ *
+ * "HTML page" is guaranteed by `content_type_refs.raw = 'text/html'`, NOT by
+ * `is_target` alone: `is_target` means "in-scope crawl target" and is set
+ * for in-scope non-HTML resources too (e.g. a PDF reached via the HEAD
+ * pre-flight is `is_target = 1`). Counting those would over-report the HTML
+ * page total, so page-ness is asserted at the read layer here rather than by
+ * trusting `is_target`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The number of `text/html` rows with `is_target = 1` and `scraped = 1`.
+ */
+export declare function getScrapedHtmlPageCount(knex: Knex): Promise<number>;

package/lib/archive/db-ops/pages/read/get-scraped-html-page-count.js

@@ -0,0 +1,25 @@
+/**
+ * Counts pages that were scraped as crawl targets (full HTML render).
+ *
+ * Used by the crawler to seed its `pagesScraped` counter on resume so the
+ * progress display reflects all browser-rendered HTML pages across sessions,
+ * not just the current one.
+ *
+ * "HTML page" is guaranteed by `content_type_refs.raw = 'text/html'`, NOT by
+ * `is_target` alone: `is_target` means "in-scope crawl target" and is set
+ * for in-scope non-HTML resources too (e.g. a PDF reached via the HEAD
+ * pre-flight is `is_target = 1`). Counting those would over-report the HTML
+ * page total, so page-ness is asserted at the read layer here rather than by
+ * trusting `is_target`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The number of `text/html` rows with `is_target = 1` and `scraped = 1`.
+ */
+export async function getScrapedHtmlPageCount(knex) {
+    const [row] = await knex('content_items')
+        .join('content_type_refs', 'content_type_refs.id', 'content_items.content_type_id')
+        .where('content_items.is_target', 1)
+        .andWhere('content_items.scraped', 1)
+        .andWhere('content_type_refs.raw', 'text/html')
+        .count('* as count');
+    return row ? Number(row.count) : 0;
+}

package/lib/archive/db-ops/pages/read/reconstruct-page-rows.d.ts

@@ -0,0 +1,31 @@
+import type { DB_Page } from '../../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Raw row shape produced by {@link ../read/build-page-query.js} before
+ * `responseHeaders` / `meta_extras` reconstruction.
+ */
+interface RawPageRow extends Omit<DB_Page, 'responseHeaders' | 'meta_extras' | 'networkLogs'> {
+    /** `content_items.header_set_id`, or null when no headers were recorded. */
+    headerSetId: number | null;
+    /** `json_refs.json_text` for `meta_extras`, or null when absent. */
+    extras_body: Buffer | string | null;
+    /** `json_refs.codec` for `meta_extras` (`'zstd'` or `'none'`), or null when absent. */
+    extras_codec: 'zstd' | 'none' | null;
+}
+/**
+ * Reconstructs `responseHeaders` (JSON string) and `meta_extras` (JSON
+ * string) for a batch of raw page rows. Headers load through
+ * {@link ../../_shared/load-response-headers-by-set-ids.js} (chunked
+ * batch lookup, never N+1); `meta_extras` decodes through
+ * {@link ../../_shared/decode-json-ref.js}. `networkLogs` has no 0.13
+ * equivalent (it is a legacy-only field nothing ever wrote past the
+ * pre-0.13 write path either) and is always `null`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param rows - Raw rows from {@link ../read/build-page-query.js}.
+ * @returns Fully reconstructed `DB_Page` rows, in the same order as `rows`.
+ * @example
+ * const raw = await buildPageQuery(knex).where('ci.is_target', 1);
+ * const pages = await reconstructPageRows(knex, raw);
+ */
+export declare function reconstructPageRows(knex: Knex, rows: readonly RawPageRow[]): Promise<DB_Page[]>;
+export {};

package/lib/archive/db-ops/pages/read/reconstruct-page-rows.js

@@ -0,0 +1,32 @@
+import { decodeJsonRef } from '../../_shared/decode-json-ref.js';
+import { loadResponseHeadersBySetIds } from '../../_shared/load-response-headers-by-set-ids.js';
+/**
+ * Reconstructs `responseHeaders` (JSON string) and `meta_extras` (JSON
+ * string) for a batch of raw page rows. Headers load through
+ * {@link ../../_shared/load-response-headers-by-set-ids.js} (chunked
+ * batch lookup, never N+1); `meta_extras` decodes through
+ * {@link ../../_shared/decode-json-ref.js}. `networkLogs` has no 0.13
+ * equivalent (it is a legacy-only field nothing ever wrote past the
+ * pre-0.13 write path either) and is always `null`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param rows - Raw rows from {@link ../read/build-page-query.js}.
+ * @returns Fully reconstructed `DB_Page` rows, in the same order as `rows`.
+ * @example
+ * const raw = await buildPageQuery(knex).where('ci.is_target', 1);
+ * const pages = await reconstructPageRows(knex, raw);
+ */
+export async function reconstructPageRows(knex, rows) {
+    const headerSetIds = [
+        ...new Set(rows.map((r) => r.headerSetId).filter((id) => id != null)),
+    ];
+    const headersBySetId = await loadResponseHeadersBySetIds(knex, headerSetIds);
+    return rows.map((row) => {
+        const { headerSetId, extras_body, extras_codec, ...rest } = row;
+        return {
+            ...rest,
+            responseHeaders: JSON.stringify(headerSetId == null ? {} : (headersBySetId.get(headerSetId) ?? {})),
+            meta_extras: decodeJsonRef(extras_body, extras_codec),
+            networkLogs: null,
+        };
+    });
+}

package/lib/archive/db-ops/pages/reset/repromote-external-pages.d.ts

@@ -0,0 +1,24 @@
+import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
+import type { Knex } from 'knex';
+/**
+ * Promote previously-external pages whose URL falls under any of the new scope
+ * entries back to a "needs scraping" state so that the next crawl picks them up
+ * as full internal pages.
+ *
+ * For each matching page:
+ * - clears the scrape metadata (status, headers, snapshot path, etc.) by
+ *   deleting the `page_meta` row outright (a re-scrape re-inserts it fresh),
+ * - flips `is_external` to `0` and `scraped` to `0` on `content_items`,
+ * - removes stale `anchor_edges`, `image_items`, and `resource_ref_edges`
+ *   rows so that the re-scrape can re-insert fresh ones without duplicates.
+ *
+ * The page row itself is kept (id is preserved) so existing referrers via
+ * `anchor_edges.href_page_id` remain valid. SELECT and UPDATE/DELETE
+ * statements are chunked to stay below SQLite's
+ * `SQLITE_LIMIT_VARIABLE_NUMBER`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param scopes - The hostname-indexed scope map after the new roots are merged.
+ * @param options - URL parsing options forwarded to {@link findScopeEntry}.
+ * @returns The URLs of the pages that were promoted.
+ */
+export declare function repromoteExternalPages(knex: Knex, scopes: ReadonlyMap<string, readonly ExURL[]>, options?: ParseURLOptions): Promise<string[]>;

package/lib/archive/db-ops/pages/reset/repromote-external-pages.js

@@ -0,0 +1,93 @@
+import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
+import { findScopeEntry } from '../../../../crawler/find-scope-entry.js';
+import { dbLog } from '../../../debug.js';
+/**
+ * Promote previously-external pages whose URL falls under any of the new scope
+ * entries back to a "needs scraping" state so that the next crawl picks them up
+ * as full internal pages.
+ *
+ * For each matching page:
+ * - clears the scrape metadata (status, headers, snapshot path, etc.) by
+ *   deleting the `page_meta` row outright (a re-scrape re-inserts it fresh),
+ * - flips `is_external` to `0` and `scraped` to `0` on `content_items`,
+ * - removes stale `anchor_edges`, `image_items`, and `resource_ref_edges`
+ *   rows so that the re-scrape can re-insert fresh ones without duplicates.
+ *
+ * The page row itself is kept (id is preserved) so existing referrers via
+ * `anchor_edges.href_page_id` remain valid. SELECT and UPDATE/DELETE
+ * statements are chunked to stay below SQLite's
+ * `SQLITE_LIMIT_VARIABLE_NUMBER`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param scopes - The hostname-indexed scope map after the new roots are merged.
+ * @param options - URL parsing options forwarded to {@link findScopeEntry}.
+ * @returns The URLs of the pages that were promoted.
+ */
+export async function repromoteExternalPages(knex, scopes, options) {
+    if (scopes.size === 0) {
+        return [];
+    }
+    const candidates = await knex('content_items')
+        .join('url_refs', 'content_items.url_id', 'url_refs.id')
+        .select('content_items.id as id', 'url_refs.url as url')
+        .where('content_items.is_external', 1);
+    const promotedIds = [];
+    const promotedUrls = [];
+    for (const row of candidates) {
+        const parsed = parseUrl(row.url, options);
+        if (!parsed) {
+            continue;
+        }
+        if (findScopeEntry(parsed, scopes, options) === null) {
+            continue;
+        }
+        promotedIds.push(row.id);
+        promotedUrls.push(row.url);
+    }
+    if (promotedIds.length === 0) {
+        return [];
+    }
+    const chunkSize = 500;
+    for (let i = 0; i < promotedIds.length; i += chunkSize) {
+        const chunk = promotedIds.slice(i, i + chunkSize);
+        await knex('content_items').whereIn('id', chunk).update({
+            scraped: 0,
+            is_external: 0,
+            is_skipped: 0,
+            skip_reason: null,
+            status: null,
+            status_text: null,
+            content_type_id: null,
+            content_length: null,
+            header_set_id: null,
+            redirect_dest_id: null,
+            // `first_crawled_at` / `last_crawled_at` are deliberately
+            // left untouched — the last-success timestamp survives the
+            // demotion.
+        });
+        // Clear the prior crawl's data for the repromoted pages. `updatePage`
+        // also replaces anchor_edges/image_items/tags/jsonld when it
+        // re-scrapes them, but only when the new scrape is non-empty — so
+        // this pre-clear is still load-bearing for pages that get
+        // repromoted but then re-scrape to nothing (or are never reached
+        // again), and it is the only place `resource_ref_edges` is cleared.
+        // Deleting the `page_meta` row (rather than nulling every column)
+        // clears title / description / og:* / twitter:* / meta_extras in
+        // one statement; a re-scrape re-inserts it via
+        // `ON CONFLICT(page_id) DO UPDATE`. `page_tags` / `page_jsonld` are
+        // cleared explicitly even though both tables also carry ON DELETE
+        // CASCADE — we keep the existing pattern of explicit chunked
+        // DELETEs rather than relying on CASCADE indirectly (and would not
+        // cascade anyway: the parent `content_items` row is updated, not
+        // deleted). Orphan blobs in `page_html_blobs` are left behind; #23
+        // will add GC.
+        await knex('page_meta').whereIn('page_id', chunk).delete();
+        await knex('anchor_edges').whereIn('page_id', chunk).delete();
+        await knex('image_items').whereIn('page_id', chunk).delete();
+        await knex('resource_ref_edges').whereIn('page_id', chunk).delete();
+        await knex('page_html_ref').whereIn('page_id', chunk).delete();
+        await knex('page_tags').whereIn('pageId', chunk).delete();
+        await knex('page_jsonld').whereIn('pageId', chunk).delete();
+    }
+    dbLog('Repromoted %d external pages back to pending', promotedUrls.length);
+    return promotedUrls;
+}

package/lib/archive/db-ops/pages/reset/reset-failed-pages.d.ts

@@ -0,0 +1,47 @@
+import type { Knex } from 'knex';
+/**
+ * Reset previously-attempted pages that ended in a recoverable failure so a
+ * follow-up crawl can re-fetch them from scratch.
+ *
+ * A page qualifies as a recoverable failure when it was already scraped
+ * (`scraped = 1`), is not a redirect source (`redirect_dest_id IS NULL`), was
+ * not intentionally skipped (`is_skipped` is not `1`), and one of the
+ * following holds:
+ *
+ * - `status = -1` — the sentinel a hard scrape failure (network error,
+ *   timeout, browser crash) is recorded with (see `handle-scrape-error.ts`);
+ * - `status IS NULL` — no status was ever stored for the row;
+ * - the row has no `content_type_refs` link — the content type could not be
+ *   determined;
+ * - `status` is in the `5xx` range — a (frequently transient) server error.
+ *
+ * Definitive `4xx` responses are intentionally excluded: re-fetching a 404
+ * almost always yields the same answer.
+ *
+ * A second exclusion runs in JS after the SQL candidate scan: any page whose
+ * latest recorded `page_errors` / `crawl_errors` message classifies into a
+ * permanent {@link PERMANENT_ERROR_KINDS} kind (dns / tls / client-blocked /
+ * parse-error / connection-refused) is left as-is rather than reset to
+ * pending. Without this filter, `--retry-failed` never converges: NXDOMAIN
+ * hosts, expired-cert hosts, and `ERR_BLOCKED_BY_CLIENT` ad pixels would be
+ * reset every iteration, re-attempted, fail identically, and rejoin the
+ * candidate pool for the next iteration. The exclusion keeps the retry
+ * target shrinking across `--retry-failed` passes by leaving deterministic
+ * dead-ends alone.
+ *
+ * Matching rows — internal and external alike — are demoted back to pending
+ * (`scraped = 0`) and have their stale scrape metadata cleared (the
+ * `page_meta` row is deleted outright rather than nulled column-by-column).
+ * The page row itself is kept (id preserved) so existing
+ * `anchor_edges.href_page_id` referrers stay valid, and `is_external` is
+ * left untouched so the next pass re-classifies each page from the crawl
+ * scope. Related `anchor_edges`, `image_items`, `resource_ref_edges`, and
+ * `page_errors` rows are deleted so the re-scrape can re-insert fresh data
+ * without duplicates.
+ *
+ * SELECT and UPDATE/DELETE statements are chunked to stay below SQLite's
+ * `SQLITE_LIMIT_VARIABLE_NUMBER`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The URLs of the pages that were reset to pending.
+ */
+export declare function resetFailedPages(knex: Knex): Promise<string[]>;

package/lib/archive/db-ops/pages/reset/reset-failed-pages.js

@@ -0,0 +1,124 @@
+import { classifyErrorKind } from '../../../../classify-error-kind.js';
+import { PERMANENT_ERROR_KINDS } from '../../../../permanent-error-kinds.js';
+import { dbLog } from '../../../debug.js';
+import { getFailedPageMessages } from '../../../get-failed-page-messages.js';
+/**
+ * Reset previously-attempted pages that ended in a recoverable failure so a
+ * follow-up crawl can re-fetch them from scratch.
+ *
+ * A page qualifies as a recoverable failure when it was already scraped
+ * (`scraped = 1`), is not a redirect source (`redirect_dest_id IS NULL`), was
+ * not intentionally skipped (`is_skipped` is not `1`), and one of the
+ * following holds:
+ *
+ * - `status = -1` — the sentinel a hard scrape failure (network error,
+ *   timeout, browser crash) is recorded with (see `handle-scrape-error.ts`);
+ * - `status IS NULL` — no status was ever stored for the row;
+ * - the row has no `content_type_refs` link — the content type could not be
+ *   determined;
+ * - `status` is in the `5xx` range — a (frequently transient) server error.
+ *
+ * Definitive `4xx` responses are intentionally excluded: re-fetching a 404
+ * almost always yields the same answer.
+ *
+ * A second exclusion runs in JS after the SQL candidate scan: any page whose
+ * latest recorded `page_errors` / `crawl_errors` message classifies into a
+ * permanent {@link PERMANENT_ERROR_KINDS} kind (dns / tls / client-blocked /
+ * parse-error / connection-refused) is left as-is rather than reset to
+ * pending. Without this filter, `--retry-failed` never converges: NXDOMAIN
+ * hosts, expired-cert hosts, and `ERR_BLOCKED_BY_CLIENT` ad pixels would be
+ * reset every iteration, re-attempted, fail identically, and rejoin the
+ * candidate pool for the next iteration. The exclusion keeps the retry
+ * target shrinking across `--retry-failed` passes by leaving deterministic
+ * dead-ends alone.
+ *
+ * Matching rows — internal and external alike — are demoted back to pending
+ * (`scraped = 0`) and have their stale scrape metadata cleared (the
+ * `page_meta` row is deleted outright rather than nulled column-by-column).
+ * The page row itself is kept (id preserved) so existing
+ * `anchor_edges.href_page_id` referrers stay valid, and `is_external` is
+ * left untouched so the next pass re-classifies each page from the crawl
+ * scope. Related `anchor_edges`, `image_items`, `resource_ref_edges`, and
+ * `page_errors` rows are deleted so the re-scrape can re-insert fresh data
+ * without duplicates.
+ *
+ * SELECT and UPDATE/DELETE statements are chunked to stay below SQLite's
+ * `SQLITE_LIMIT_VARIABLE_NUMBER`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The URLs of the pages that were reset to pending.
+ */
+export async function resetFailedPages(knex) {
+    const candidates = await knex('content_items')
+        .join('url_refs', 'content_items.url_id', 'url_refs.id')
+        .select('content_items.id as id', 'url_refs.url as url')
+        .where('content_items.scraped', 1)
+        .whereNull('content_items.redirect_dest_id')
+        .where((qb) => {
+        qb.where('content_items.is_skipped', 0).orWhereNull('content_items.is_skipped');
+    })
+        .where((qb) => {
+        qb.whereNull('content_items.status')
+            .orWhere('content_items.status', -1)
+            .orWhereNull('content_items.content_type_id')
+            .orWhereBetween('content_items.status', [500, 599]);
+    });
+    if (candidates.length === 0) {
+        return [];
+    }
+    const candidateIds = candidates.map((row) => row.id);
+    const candidateUrls = candidates.map((row) => row.url);
+    const messages = await getFailedPageMessages(knex, candidateIds, candidateUrls);
+    // Drop candidates whose latest recorded message classifies as permanent.
+    // An empty/absent message stays in the retry pool — we keep retrying when
+    // we don't know it's permanent, erring on the side of investigation.
+    const retryable = candidates.filter((row) => {
+        const message = messages.get(row.id) ?? '';
+        if (message === '') {
+            return true;
+        }
+        return !PERMANENT_ERROR_KINDS.has(classifyErrorKind(message));
+    });
+    const excludedCount = candidates.length - retryable.length;
+    if (excludedCount > 0) {
+        dbLog('Excluded %d page(s) from retry — permanent failure kinds (dns/tls/client-blocked/parse-error/connection-refused)', excludedCount);
+    }
+    if (retryable.length === 0) {
+        return [];
+    }
+    const ids = retryable.map((row) => row.id);
+    const urls = retryable.map((row) => row.url);
+    const chunkSize = 500;
+    for (let i = 0; i < ids.length; i += chunkSize) {
+        const chunk = ids.slice(i, i + chunkSize);
+        await knex('content_items').whereIn('id', chunk).update({
+            scraped: 0,
+            status: null,
+            status_text: null,
+            content_type_id: null,
+            content_length: null,
+            header_set_id: null,
+            // `first_crawled_at` / `last_crawled_at` are deliberately left
+            // untouched so the last-success timestamp records survive the
+            // demotion (the within-archive observation axis for #11/#17/#19).
+        });
+        // Clear the prior crawl's per-page data so the re-scrape starts clean.
+        // `updatePage` only replaces anchor_edges/image_items/tags/jsonld when
+        // the new scrape is non-empty, so this pre-clear is load-bearing for
+        // pages that reset but then fail again (or are never reached), and it
+        // is the only place `resource_ref_edges` and `page_errors` are
+        // cleared. Deleting the `page_meta` row (rather than nulling every
+        // column) clears title / description / og:* / twitter:* /
+        // meta_extras in one statement; a re-scrape re-inserts it via
+        // `ON CONFLICT(page_id) DO UPDATE`.
+        await knex('page_meta').whereIn('page_id', chunk).delete();
+        await knex('anchor_edges').whereIn('page_id', chunk).delete();
+        await knex('image_items').whereIn('page_id', chunk).delete();
+        await knex('resource_ref_edges').whereIn('page_id', chunk).delete();
+        await knex('page_errors').whereIn('pageId', chunk).delete();
+        await knex('page_html_ref').whereIn('page_id', chunk).delete();
+        await knex('page_tags').whereIn('pageId', chunk).delete();
+        await knex('page_jsonld').whereIn('pageId', chunk).delete();
+    }
+    dbLog('Reset %d failed pages back to pending', urls.length);
+    return urls;
+}

package/lib/archive/db-ops/pages/write/insert-inventory-seeds.d.ts

@@ -0,0 +1,37 @@
+import type { WriteRefCaches } from '../../_shared/types.js';
+import type { Knex } from 'knex';
+/**
+ * Pre-insert inventory HTML seeds into `content_items` as `scraped = 0`,
+ * `source = 'inventory-seed'` placeholders so the URL's existence in the
+ * archive is **durable before the scrape phase starts**.
+ *
+ * Why this is the linchpin of `--inventory` Ctrl+C tolerance: without
+ * pre-insertion, HTML seeds live only in the Crawler's in-memory
+ * `LinkList` until the dealer eventually calls `setPage`. A Ctrl+C /
+ * crash before that point loses the seed without trace, and `--resume`
+ * cannot recover it because `getCrawlingState`'s strict pending set
+ * requires a `content_items` row. Pre-inserting fills exactly that gap:
+ * the strict pending set picks these rows up via its
+ * `OR source != 'crawled'` clause, so `--resume` after an interrupted
+ * inventory pass picks every seed back up. See `getCrawlingState` for
+ * the strict-set rationale.
+ *
+ * Idempotent: both the `url_refs` and `content_items` inserts are
+ * `ON CONFLICT ... IGNORE`, keeping existing rows intact. The
+ * `resolveContentItemId` crawled-wins downgrade still fires later when a
+ * crawled-lineage anchor reaches one of these seeds — that's the right
+ * behaviour (a seed that turned out to be reachable is not an orphan
+ * and should not retain the inventory label).
+ *
+ * Chunked into 500-URL batches so SQLite's bound-parameter limit
+ * (`SQLITE_MAX_VARIABLE_NUMBER`) cannot be hit even on a
+ * tens-of-thousands inventory list.
+ *
+ * Called by `CrawlerOrchestrator.inventory` during the
+ * `.bak`-protected ingestion phase, so any failure here aborts the run
+ * and restores from backup — the operator reruns from scratch.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches
+ * @param urls - URL strings already in `withoutHashAndAuth` form.
+ */
+export declare function insertInventorySeeds(knex: Knex, caches: WriteRefCaches, urls: readonly string[]): Promise<void>;

package/lib/archive/db-ops/pages/write/insert-inventory-seeds.js

@@ -0,0 +1,72 @@
+import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+import { resolveUrlRefs } from '../../../populate-entity-tables/resolve-url-refs.js';
+import { decomposeUrl } from '../../../populate-ref-tables/decompose-url.js';
+/**
+ * Pre-insert inventory HTML seeds into `content_items` as `scraped = 0`,
+ * `source = 'inventory-seed'` placeholders so the URL's existence in the
+ * archive is **durable before the scrape phase starts**.
+ *
+ * Why this is the linchpin of `--inventory` Ctrl+C tolerance: without
+ * pre-insertion, HTML seeds live only in the Crawler's in-memory
+ * `LinkList` until the dealer eventually calls `setPage`. A Ctrl+C /
+ * crash before that point loses the seed without trace, and `--resume`
+ * cannot recover it because `getCrawlingState`'s strict pending set
+ * requires a `content_items` row. Pre-inserting fills exactly that gap:
+ * the strict pending set picks these rows up via its
+ * `OR source != 'crawled'` clause, so `--resume` after an interrupted
+ * inventory pass picks every seed back up. See `getCrawlingState` for
+ * the strict-set rationale.
+ *
+ * Idempotent: both the `url_refs` and `content_items` inserts are
+ * `ON CONFLICT ... IGNORE`, keeping existing rows intact. The
+ * `resolveContentItemId` crawled-wins downgrade still fires later when a
+ * crawled-lineage anchor reaches one of these seeds — that's the right
+ * behaviour (a seed that turned out to be reachable is not an orphan
+ * and should not retain the inventory label).
+ *
+ * Chunked into 500-URL batches so SQLite's bound-parameter limit
+ * (`SQLITE_MAX_VARIABLE_NUMBER`) cannot be hit even on a
+ * tens-of-thousands inventory list.
+ *
+ * Called by `CrawlerOrchestrator.inventory` during the
+ * `.bak`-protected ingestion phase, so any failure here aborts the run
+ * and restores from backup — the operator reruns from scratch.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches
+ * @param urls - URL strings already in `withoutHashAndAuth` form.
+ */
+export async function insertInventorySeeds(knex, caches, urls) {
+    if (urls.length === 0) {
+        return;
+    }
+    await eachSplitted([...urls], 500, async (chunk) => {
+        await knex('url_refs')
+            .insert(chunk.map((url) => ({ url, ...decomposeUrl(url) })))
+            .onConflict('url')
+            .ignore();
+        const urlIds = await resolveUrlRefs(knex, chunk);
+        const rows = chunk.map((url) => {
+            const urlId = urlIds.get(url);
+            if (urlId === undefined) {
+                throw new Error(`insertInventorySeeds: url_refs.id not resolved for ${url}`);
+            }
+            caches.urlIds.set(url, urlId);
+            return {
+                url_id: urlId,
+                scraped: 0,
+                is_external: 0,
+                is_target: 0,
+                source: 'inventory-seed',
+            };
+        });
+        await knex('content_items').insert(rows).onConflict('url_id').ignore();
+        const inserted = (await knex
+            .select('ci.id', 'ci.source', 'ur.url')
+            .from('content_items as ci')
+            .join('url_refs as ur', 'ur.id', 'ci.url_id')
+            .whereIn('ur.url', chunk));
+        for (const row of inserted) {
+            caches.contentItems.set(row.url, { id: row.id, source: row.source });
+        }
+    });
+}

package/lib/archive/db-ops/pages/write/insert-jsonld.d.ts

@@ -0,0 +1,17 @@
+import type { PageData } from '../../../../utils/types/types.js';
+import type { Knex } from 'knex';
+/**
+ * Replaces the page's JSON-LD / SpeculationRules rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction.
+ *
+ * `writeHtml = false` branches (`setExternalPage`, metadata-only) skip
+ * this entirely — JSON-LD lives inside the HTML body, so external pages
+ * that are not rendered have no entries to write. An empty array on a
+ * normally-rendered page is treated as a degraded re-scrape: prior rows
+ * are kept (same `delete-only-when-replacing` invariant as `anchors` /
+ * `images`).
+ * @param pageId
+ * @param meta
+ * @param trx
+ */
+export declare function insertJsonLd(pageId: number, meta: PageData['meta'], trx: Knex.Transaction): Promise<void>;

package/lib/archive/db-ops/pages/write/insert-jsonld.js

@@ -0,0 +1,49 @@
+import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+import { classifyJsonLdType } from '../../../meta/classify-jsonld-type.js';
+/**
+ * Replaces the page's JSON-LD / SpeculationRules rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction.
+ *
+ * `writeHtml = false` branches (`setExternalPage`, metadata-only) skip
+ * this entirely — JSON-LD lives inside the HTML body, so external pages
+ * that are not rendered have no entries to write. An empty array on a
+ * normally-rendered page is treated as a degraded re-scrape: prior rows
+ * are kept (same `delete-only-when-replacing` invariant as `anchors` /
+ * `images`).
+ * @param pageId
+ * @param meta
+ * @param trx
+ */
+export async function insertJsonLd(pageId, meta, trx) {
+    // `??` guards tolerate the legacy "minimal meta" shape from older test
+    // fixtures. Real beholder 3.0.0 always populates these required fields.
+    const jsonLd = meta.jsonLd ?? [];
+    const speculationRules = meta.speculationRules ?? [];
+    const rows = [];
+    for (const entry of jsonLd) {
+        rows.push({
+            pageId,
+            kind: 'ld+json',
+            type: classifyJsonLdType(entry),
+            raw: entry.raw,
+            parsed: entry.parsed === undefined ? null : JSON.stringify(entry.parsed),
+            parseError: entry.parseError ?? null,
+        });
+    }
+    for (const entry of speculationRules) {
+        rows.push({
+            pageId,
+            kind: 'speculationrules',
+            type: classifyJsonLdType(entry),
+            raw: entry.raw,
+            parsed: entry.parsed === undefined ? null : JSON.stringify(entry.parsed),
+            parseError: entry.parseError ?? null,
+        });
+    }
+    if (rows.length === 0)
+        return;
+    await trx('page_jsonld').where('pageId', pageId).delete();
+    await eachSplitted(rows, 100, async (chunk) => {
+        await trx('page_jsonld').insert(chunk);
+    });
+}