@nitpicker/crawler 0.11.0 → 0.13.0

package/lib/archive/populate-entity-tables/resolve-content-type-refs.js

@@ -0,0 +1,27 @@
+/**
+ * Loads the entire `content_type_refs` dictionary into an in-process map
+ * keyed by the raw wire value (issue #193).
+ *
+ * Content-type cardinality is small in practice — the reference archive
+ * carries ≈ 400 distinct raw values across `pages` + `resources`, so
+ * loading everything into a `Map` costs ≈ 40 KB and eliminates every
+ * per-chunk round-trip. This is the same trade-off {@link
+ * ../populate-ref-tables/populate-header-tables.ts} makes for `header_name_refs`.
+ *
+ * Callers should invoke this once at the start of a 0.13 populate
+ * step (e.g. `populateContentItems`, `populateResourceItems`) and reuse
+ * the map for every chunk.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @returns Map keyed by `content_type_refs.raw`.
+ * @example
+ * const contentTypeIds = await loadContentTypeRefs(trx);
+ * const id = contentTypeIds.get('text/html; charset=utf-8'); // number | undefined
+ */
+export async function loadContentTypeRefs(trx) {
+    const rows = await trx('content_type_refs').select('id', 'raw');
+    const map = new Map();
+    for (const row of rows) {
+        map.set(row.raw, row.id);
+    }
+    return map;
+}

package/lib/archive/populate-entity-tables/resolve-header-sets.d.ts

@@ -0,0 +1,49 @@
+import type { HeaderSetIdMap } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Batch-resolves `header_sets.id` for a set of raw `responseHeaders`
+ * JSON strings (issue #193).
+ *
+ * The lookup is a two-stage cascade:
+ *
+ * 1. **Primary lookup** by `header_sets.raw_json_hash` — SHA-256 of the
+ *    raw JSON string exactly as stored on the source row. This hits when
+ *    the row's JSON key ordering matches whichever ordering populated
+ *    `header_sets` first for that stable-set equivalence class (see
+ *    {@link ../populate-ref-tables/populate-header-tables.ts}).
+ * 2. **Fallback lookup** by `header_sets.raw_hash` for the misses — SHA-
+ *    256 of the sorted `name=value` pairs. `populate-header-tables.ts`
+ *    stores exactly one `raw_json_hash` per `raw_hash` equivalence class
+ *    (the first JSON variant encountered), so a second variant with
+ *    identical decoded content but different key ordering never had its
+ *    `raw_json_hash` persisted — but its `raw_hash` still points at the
+ *    shared `header_sets` row. Without the fallback these rows would
+ *    silently get `header_set_id = NULL` on `content_items` /
+ *    `resource_items` and the count-based acceptance check would still
+ *    pass; the fallback closes that gap.
+ *
+ * Hashes are computed in JS because SQLite has no built-in BLAKE3 /
+ * SHA-256 (see {@link ../populate-ref-tables/compute-content-hash.ts} for the
+ * algorithm choice). The `raw_hash` fallback re-decomposes each miss via
+ * {@link ../populate-ref-tables/decompose-header-set.ts} so the canonicalisation
+ * matches the one `populateHeaderTables` used at insert time.
+ *
+ * Empty / null values and the sentinel `'{}'` / `'null'` strings are
+ * skipped — those responses were decomposed to `null` by
+ * `decomposeHeaderSet` and never produced a `header_sets` row, so the
+ * caller writes `header_set_id = null` for them.
+ *
+ * Duplicate JSON strings are deduped internally so a page and a
+ * resource sharing the same raw responseHeaders JSON only contribute
+ * one lookup slot.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param rawJsonStrings - Iterable of raw responseHeaders JSON strings.
+ * @returns Map keyed by the raw JSON string; missing entries indicate
+ *   no matching `header_sets` row exists (parse failure or empty set).
+ * @example
+ * const idMap = await resolveHeaderSets(trx, [
+ *   '{"content-type":"text/html"}',
+ * ]);
+ * const setId = idMap.get('{"content-type":"text/html"}'); // number | undefined
+ */
+export declare function resolveHeaderSets(trx: Knex, rawJsonStrings: Iterable<string>): Promise<HeaderSetIdMap>;

package/lib/archive/populate-entity-tables/resolve-header-sets.js

@@ -0,0 +1,122 @@
+import { computeContentHash } from '../populate-ref-tables/compute-content-hash.js';
+import { decomposeHeaderSet } from '../populate-ref-tables/decompose-header-set.js';
+/**
+ * Rows sent per `SELECT ... WHERE raw_json_hash IN (?, ...)` chunk. Same
+ * rationale as {@link ./resolve-url-refs.ts}: 500 rows × 1 param each is
+ * well under the SQLite variable limit and gives good round-trip
+ * amortisation.
+ */
+const LOOKUP_CHUNK_SIZE = 500;
+/**
+ * Batch-resolves `header_sets.id` for a set of raw `responseHeaders`
+ * JSON strings (issue #193).
+ *
+ * The lookup is a two-stage cascade:
+ *
+ * 1. **Primary lookup** by `header_sets.raw_json_hash` — SHA-256 of the
+ *    raw JSON string exactly as stored on the source row. This hits when
+ *    the row's JSON key ordering matches whichever ordering populated
+ *    `header_sets` first for that stable-set equivalence class (see
+ *    {@link ../populate-ref-tables/populate-header-tables.ts}).
+ * 2. **Fallback lookup** by `header_sets.raw_hash` for the misses — SHA-
+ *    256 of the sorted `name=value` pairs. `populate-header-tables.ts`
+ *    stores exactly one `raw_json_hash` per `raw_hash` equivalence class
+ *    (the first JSON variant encountered), so a second variant with
+ *    identical decoded content but different key ordering never had its
+ *    `raw_json_hash` persisted — but its `raw_hash` still points at the
+ *    shared `header_sets` row. Without the fallback these rows would
+ *    silently get `header_set_id = NULL` on `content_items` /
+ *    `resource_items` and the count-based acceptance check would still
+ *    pass; the fallback closes that gap.
+ *
+ * Hashes are computed in JS because SQLite has no built-in BLAKE3 /
+ * SHA-256 (see {@link ../populate-ref-tables/compute-content-hash.ts} for the
+ * algorithm choice). The `raw_hash` fallback re-decomposes each miss via
+ * {@link ../populate-ref-tables/decompose-header-set.ts} so the canonicalisation
+ * matches the one `populateHeaderTables` used at insert time.
+ *
+ * Empty / null values and the sentinel `'{}'` / `'null'` strings are
+ * skipped — those responses were decomposed to `null` by
+ * `decomposeHeaderSet` and never produced a `header_sets` row, so the
+ * caller writes `header_set_id = null` for them.
+ *
+ * Duplicate JSON strings are deduped internally so a page and a
+ * resource sharing the same raw responseHeaders JSON only contribute
+ * one lookup slot.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param rawJsonStrings - Iterable of raw responseHeaders JSON strings.
+ * @returns Map keyed by the raw JSON string; missing entries indicate
+ *   no matching `header_sets` row exists (parse failure or empty set).
+ * @example
+ * const idMap = await resolveHeaderSets(trx, [
+ *   '{"content-type":"text/html"}',
+ * ]);
+ * const setId = idMap.get('{"content-type":"text/html"}'); // number | undefined
+ */
+export async function resolveHeaderSets(trx, rawJsonStrings) {
+    const distinct = new Set();
+    for (const raw of rawJsonStrings) {
+        if (typeof raw !== 'string' || raw === '' || raw === 'null' || raw === '{}') {
+            continue;
+        }
+        distinct.add(raw);
+    }
+    if (distinct.size === 0) {
+        return new Map();
+    }
+    const values = [...distinct];
+    const hashByValue = new Map();
+    const hashHexToValue = new Map();
+    for (const value of values) {
+        const hash = computeContentHash(value);
+        hashByValue.set(value, hash);
+        hashHexToValue.set(hash.toString('hex'), value);
+    }
+    const hashes = [...hashByValue.values()];
+    const result = new Map();
+    for (let index = 0; index < hashes.length; index += LOOKUP_CHUNK_SIZE) {
+        const chunkHashes = hashes.slice(index, index + LOOKUP_CHUNK_SIZE);
+        const rows = await trx('header_sets')
+            .select('id', 'raw_json_hash')
+            .whereIn('raw_json_hash', chunkHashes);
+        for (const row of rows) {
+            const hex = Buffer.from(row.raw_json_hash).toString('hex');
+            const value = hashHexToValue.get(hex);
+            if (value !== undefined) {
+                result.set(value, row.id);
+            }
+        }
+    }
+    // Fallback pass: any value we could not resolve by `raw_json_hash`
+    // might still map to an existing `header_sets` row via `raw_hash`
+    // (identical decoded content, different JSON key ordering).
+    const misses = values.filter((value) => !result.has(value));
+    if (misses.length === 0) {
+        return result;
+    }
+    const rawHashByValue = new Map();
+    const rawHashHexToValue = new Map();
+    for (const value of misses) {
+        const decomposed = decomposeHeaderSet(value);
+        if (decomposed === null) {
+            continue;
+        }
+        rawHashByValue.set(value, decomposed.rawHash);
+        rawHashHexToValue.set(decomposed.rawHash.toString('hex'), value);
+    }
+    const rawHashes = [...rawHashByValue.values()];
+    for (let index = 0; index < rawHashes.length; index += LOOKUP_CHUNK_SIZE) {
+        const chunkHashes = rawHashes.slice(index, index + LOOKUP_CHUNK_SIZE);
+        const rows = await trx('header_sets')
+            .select('id', 'raw_hash')
+            .whereIn('raw_hash', chunkHashes);
+        for (const row of rows) {
+            const hex = Buffer.from(row.raw_hash).toString('hex');
+            const value = rawHashHexToValue.get(hex);
+            if (value !== undefined) {
+                result.set(value, row.id);
+            }
+        }
+    }
+    return result;
+}

package/lib/archive/populate-entity-tables/resolve-json-refs.d.ts

@@ -0,0 +1,25 @@
+import type { Knex } from 'knex';
+/**
+ * Batch-resolves `json_refs.id` for a set of raw `meta_extras` JSON
+ * strings (issue #193).
+ *
+ * The lookup goes through `json_refs.hash`, the SHA-256 hash of the raw
+ * JSON string stored by `populateJsonRefs` (see
+ * {@link ../populate-ref-tables/populate-json-refs.ts}). Hashes are computed in JS
+ * for the same reason as {@link ./resolve-header-sets.ts}: SQLite has no
+ * built-in hash function.
+ *
+ * Empty / null strings are skipped — those pages had no `meta_extras`
+ * and never produced a `json_refs` row, so the caller writes
+ * `meta_extras_json_id = null` for them.
+ *
+ * Duplicate strings in `rawJsonStrings` are deduped internally.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param rawJsonStrings - Iterable of raw `meta_extras` JSON strings.
+ * @returns Map keyed by the raw JSON string; missing entries indicate
+ *   no matching `json_refs` row exists.
+ * @example
+ * const idMap = await resolveJsonRefs(trx, ['{"customField":"x"}']);
+ * const jsonId = idMap.get('{"customField":"x"}'); // number | undefined
+ */
+export declare function resolveJsonRefs(trx: Knex, rawJsonStrings: Iterable<string>): Promise<ReadonlyMap<string, number>>;

package/lib/archive/populate-entity-tables/resolve-json-refs.js

@@ -0,0 +1,67 @@
+import { computeContentHash } from '../populate-ref-tables/compute-content-hash.js';
+/**
+ * Rows sent per `SELECT ... WHERE hash IN (?, ...)` chunk. `json_refs`
+ * cardinality is bounded by the count of distinct `pages.meta_extras`
+ * JSON strings — typically low thousands on a reference archive — so
+ * 500 per chunk keeps round-trips low without straining the SQLite
+ * variable limit.
+ */
+const LOOKUP_CHUNK_SIZE = 500;
+/**
+ * Batch-resolves `json_refs.id` for a set of raw `meta_extras` JSON
+ * strings (issue #193).
+ *
+ * The lookup goes through `json_refs.hash`, the SHA-256 hash of the raw
+ * JSON string stored by `populateJsonRefs` (see
+ * {@link ../populate-ref-tables/populate-json-refs.ts}). Hashes are computed in JS
+ * for the same reason as {@link ./resolve-header-sets.ts}: SQLite has no
+ * built-in hash function.
+ *
+ * Empty / null strings are skipped — those pages had no `meta_extras`
+ * and never produced a `json_refs` row, so the caller writes
+ * `meta_extras_json_id = null` for them.
+ *
+ * Duplicate strings in `rawJsonStrings` are deduped internally.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param rawJsonStrings - Iterable of raw `meta_extras` JSON strings.
+ * @returns Map keyed by the raw JSON string; missing entries indicate
+ *   no matching `json_refs` row exists.
+ * @example
+ * const idMap = await resolveJsonRefs(trx, ['{"customField":"x"}']);
+ * const jsonId = idMap.get('{"customField":"x"}'); // number | undefined
+ */
+export async function resolveJsonRefs(trx, rawJsonStrings) {
+    const distinct = new Set();
+    for (const raw of rawJsonStrings) {
+        if (typeof raw === 'string' && raw !== '') {
+            distinct.add(raw);
+        }
+    }
+    if (distinct.size === 0) {
+        return new Map();
+    }
+    const values = [...distinct];
+    const hashByValue = new Map();
+    const hashHexToValue = new Map();
+    for (const value of values) {
+        const hash = computeContentHash(value);
+        hashByValue.set(value, hash);
+        hashHexToValue.set(hash.toString('hex'), value);
+    }
+    const hashes = [...hashByValue.values()];
+    const result = new Map();
+    for (let index = 0; index < hashes.length; index += LOOKUP_CHUNK_SIZE) {
+        const chunkHashes = hashes.slice(index, index + LOOKUP_CHUNK_SIZE);
+        const rows = await trx('json_refs')
+            .select('id', 'hash')
+            .whereIn('hash', chunkHashes);
+        for (const row of rows) {
+            const hex = Buffer.from(row.hash).toString('hex');
+            const value = hashHexToValue.get(hex);
+            if (value !== undefined) {
+                result.set(value, row.id);
+            }
+        }
+    }
+    return result;
+}

package/lib/archive/populate-entity-tables/resolve-text-refs.d.ts

@@ -0,0 +1,30 @@
+import type { TextRefIdMap } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Batch-resolves `text_refs.id` for a set of raw text strings (issue #193).
+ *
+ * The `text_refs` UNIQUE constraint is on `(hash, text)` so lookups
+ * prefix-seek on `hash`. Each caller-supplied text is hashed in JS via
+ * {@link ../populate-ref-tables/compute-content-hash.ts} (32-byte SHA-256, matching
+ * how `populateTextRefs` inserted the rows) before the
+ * SQL query; providing the hash lets SQLite's index seek in O(log n)
+ * without a full table scan.
+ *
+ * The `(hash, text)` UNIQUE composite makes the trailing text column
+ * theoretically necessary to disambiguate the astronomically improbable
+ * hash collision — we narrow on both by including `text` in the WHERE
+ * clause, then filter the map keys by the exact text on read-back.
+ *
+ * Empty / null texts are ignored. Duplicate strings in `texts` are
+ * deduped internally.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param texts - Iterable of raw text strings to resolve.
+ * @returns Map keyed by the raw text; missing entries indicate the text
+ *   is not present in `text_refs` (which should not happen after
+ *   `populateTextRefs` completes but does happen mid-migration when a caller
+ *   passes texts that were never inserted).
+ * @example
+ * const idMap = await resolveTextRefs(trx, ['My Page Title', 'Alt text']);
+ * const titleId = idMap.get('My Page Title'); // number | undefined
+ */
+export declare function resolveTextRefs(trx: Knex, texts: Iterable<string>): Promise<TextRefIdMap>;

package/lib/archive/populate-entity-tables/resolve-text-refs.js

@@ -0,0 +1,61 @@
+import { computeContentHash } from '../populate-ref-tables/compute-content-hash.js';
+/**
+ * Rows sent per `SELECT ... WHERE hash IN (?, ...)` chunk. Same rationale
+ * as {@link ./resolve-url-refs.ts}: 800 rows keeps the parameter count
+ * well under the SQLite variable limit even in the worst case where every
+ * hash is bound as its own parameter.
+ */
+const LOOKUP_CHUNK_SIZE = 800;
+/**
+ * Batch-resolves `text_refs.id` for a set of raw text strings (issue #193).
+ *
+ * The `text_refs` UNIQUE constraint is on `(hash, text)` so lookups
+ * prefix-seek on `hash`. Each caller-supplied text is hashed in JS via
+ * {@link ../populate-ref-tables/compute-content-hash.ts} (32-byte SHA-256, matching
+ * how `populateTextRefs` inserted the rows) before the
+ * SQL query; providing the hash lets SQLite's index seek in O(log n)
+ * without a full table scan.
+ *
+ * The `(hash, text)` UNIQUE composite makes the trailing text column
+ * theoretically necessary to disambiguate the astronomically improbable
+ * hash collision — we narrow on both by including `text` in the WHERE
+ * clause, then filter the map keys by the exact text on read-back.
+ *
+ * Empty / null texts are ignored. Duplicate strings in `texts` are
+ * deduped internally.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param texts - Iterable of raw text strings to resolve.
+ * @returns Map keyed by the raw text; missing entries indicate the text
+ *   is not present in `text_refs` (which should not happen after
+ *   `populateTextRefs` completes but does happen mid-migration when a caller
+ *   passes texts that were never inserted).
+ * @example
+ * const idMap = await resolveTextRefs(trx, ['My Page Title', 'Alt text']);
+ * const titleId = idMap.get('My Page Title'); // number | undefined
+ */
+export async function resolveTextRefs(trx, texts) {
+    const distinct = new Set();
+    for (const text of texts) {
+        if (typeof text === 'string' && text !== '') {
+            distinct.add(text);
+        }
+    }
+    if (distinct.size === 0) {
+        return new Map();
+    }
+    const values = [...distinct];
+    const hashes = values.map((text) => computeContentHash(text));
+    const result = new Map();
+    for (let index = 0; index < values.length; index += LOOKUP_CHUNK_SIZE) {
+        const chunkValues = values.slice(index, index + LOOKUP_CHUNK_SIZE);
+        const chunkHashes = hashes.slice(index, index + LOOKUP_CHUNK_SIZE);
+        const rows = await trx('text_refs')
+            .select('id', 'hash', 'text')
+            .whereIn('hash', chunkHashes)
+            .whereIn('text', chunkValues);
+        for (const row of rows) {
+            result.set(row.text, row.id);
+        }
+    }
+    return result;
+}

package/lib/archive/populate-entity-tables/resolve-url-or-blob-from-maps.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Routes one legacy URL-shaped value to either `url_refs` or `blob_refs`
+ * per the {@link isBlobRefValue} threshold rule, looking the id up in
+ * pre-resolved maps rather than upserting (the populate steps batch-resolve
+ * every distinct value across a whole chunk before this per-row lookup —
+ * see `resolveUrlRefs` / `resolveBlobRefs`). At most one of `url` / `blob`
+ * is non-null; both may be `null` when the value is null or fails to
+ * resolve (e.g. the `blob_refs` row is missing because the data URI was
+ * malformed and skipped by `populateBlobRefs`).
+ * @param value - Raw URL-shaped column value.
+ * @param urlIds - Map of URL string → `url_refs.id`.
+ * @param blobIds - Map of data-URI string → `blob_refs.id`.
+ * @returns `{ url, blob }` pair with at most one non-null field.
+ * @example
+ * const slot = resolveUrlOrBlobFromMaps(row.src, urlIds, blobIds);
+ * // slot.url set for a regular URL, slot.blob set for a large data: URI.
+ */
+export declare function resolveUrlOrBlobFromMaps(value: string | null, urlIds: ReadonlyMap<string, number>, blobIds: ReadonlyMap<string, number>): {
+    url: number | null;
+    blob: number | null;
+};

package/lib/archive/populate-entity-tables/resolve-url-or-blob-from-maps.js

@@ -0,0 +1,27 @@
+import { isBlobRefValue } from './is-blob-ref-value.js';
+/**
+ * Routes one legacy URL-shaped value to either `url_refs` or `blob_refs`
+ * per the {@link isBlobRefValue} threshold rule, looking the id up in
+ * pre-resolved maps rather than upserting (the populate steps batch-resolve
+ * every distinct value across a whole chunk before this per-row lookup —
+ * see `resolveUrlRefs` / `resolveBlobRefs`). At most one of `url` / `blob`
+ * is non-null; both may be `null` when the value is null or fails to
+ * resolve (e.g. the `blob_refs` row is missing because the data URI was
+ * malformed and skipped by `populateBlobRefs`).
+ * @param value - Raw URL-shaped column value.
+ * @param urlIds - Map of URL string → `url_refs.id`.
+ * @param blobIds - Map of data-URI string → `blob_refs.id`.
+ * @returns `{ url, blob }` pair with at most one non-null field.
+ * @example
+ * const slot = resolveUrlOrBlobFromMaps(row.src, urlIds, blobIds);
+ * // slot.url set for a regular URL, slot.blob set for a large data: URI.
+ */
+export function resolveUrlOrBlobFromMaps(value, urlIds, blobIds) {
+    if (typeof value !== 'string' || value === '') {
+        return { url: null, blob: null };
+    }
+    if (isBlobRefValue(value)) {
+        return { url: null, blob: blobIds.get(value) ?? null };
+    }
+    return { url: urlIds.get(value) ?? null, blob: null };
+}

package/lib/archive/populate-entity-tables/resolve-url-refs.d.ts

@@ -0,0 +1,33 @@
+import type { UrlRefIdMap } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Batch-resolves `url_refs.id` for a set of URL strings (issue #193).
+ *
+ * Called by every entity populate helper that needs to translate a
+ * legacy URL string column into a `url_id` FK. Rather than open a
+ * separate SELECT per row (which multiplies migration wall-clock by
+ * chunk size × URL column count) the resolver collects the distinct
+ * strings the caller cares about, splits them into chunks of
+ * {@link LOOKUP_CHUNK_SIZE}, issues one `WHERE url IN (?, ...)` per
+ * chunk, and returns a single map.
+ *
+ * Missing entries fall out of the map — callers distinguish "URL is a
+ * large data URI routed to blob_refs" from "URL is absent from the
+ * dictionary" via a separate {@link ./resolve-blob-refs.ts} probe.
+ *
+ * Duplicates in `urls` are deduped internally so an image whose `src`
+ * and `currentSrc` are the same value only counts once against
+ * `LOOKUP_CHUNK_SIZE`.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param urls - Iterable of URL strings to resolve. Empty and null-like
+ *   values are ignored.
+ * @returns Map keyed by the raw URL string; missing entries indicate
+ *   the URL is not present in `url_refs`.
+ * @example
+ * const idMap = await resolveUrlRefs(trx, [
+ *   'https://example.com/a',
+ *   'https://example.com/b',
+ * ]);
+ * const id = idMap.get('https://example.com/a'); // number | undefined
+ */
+export declare function resolveUrlRefs(trx: Knex, urls: Iterable<string>): Promise<UrlRefIdMap>;

package/lib/archive/populate-entity-tables/resolve-url-refs.js

@@ -0,0 +1,60 @@
+/**
+ * Rows sent per `SELECT ... WHERE url IN (?, ...)` chunk. SQLite's default
+ * `SQLITE_MAX_VARIABLE_NUMBER` is 32766 on modern builds — 800 URLs per
+ * chunk stays well under that while amortising the round-trip cost across
+ * many rows.
+ */
+const LOOKUP_CHUNK_SIZE = 800;
+/**
+ * Batch-resolves `url_refs.id` for a set of URL strings (issue #193).
+ *
+ * Called by every entity populate helper that needs to translate a
+ * legacy URL string column into a `url_id` FK. Rather than open a
+ * separate SELECT per row (which multiplies migration wall-clock by
+ * chunk size × URL column count) the resolver collects the distinct
+ * strings the caller cares about, splits them into chunks of
+ * {@link LOOKUP_CHUNK_SIZE}, issues one `WHERE url IN (?, ...)` per
+ * chunk, and returns a single map.
+ *
+ * Missing entries fall out of the map — callers distinguish "URL is a
+ * large data URI routed to blob_refs" from "URL is absent from the
+ * dictionary" via a separate {@link ./resolve-blob-refs.ts} probe.
+ *
+ * Duplicates in `urls` are deduped internally so an image whose `src`
+ * and `currentSrc` are the same value only counts once against
+ * `LOOKUP_CHUNK_SIZE`.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param urls - Iterable of URL strings to resolve. Empty and null-like
+ *   values are ignored.
+ * @returns Map keyed by the raw URL string; missing entries indicate
+ *   the URL is not present in `url_refs`.
+ * @example
+ * const idMap = await resolveUrlRefs(trx, [
+ *   'https://example.com/a',
+ *   'https://example.com/b',
+ * ]);
+ * const id = idMap.get('https://example.com/a'); // number | undefined
+ */
+export async function resolveUrlRefs(trx, urls) {
+    const distinct = new Set();
+    for (const url of urls) {
+        if (typeof url === 'string' && url !== '') {
+            distinct.add(url);
+        }
+    }
+    if (distinct.size === 0) {
+        return new Map();
+    }
+    const values = [...distinct];
+    const result = new Map();
+    for (let index = 0; index < values.length; index += LOOKUP_CHUNK_SIZE) {
+        const chunk = values.slice(index, index + LOOKUP_CHUNK_SIZE);
+        const rows = await trx('url_refs')
+            .select('id', 'url')
+            .whereIn('url', chunk);
+        for (const row of rows) {
+            result.set(row.url, row.id);
+        }
+    }
+    return result;
+}

package/lib/archive/populate-entity-tables/test-utils/count-rows.d.ts

@@ -0,0 +1,17 @@
+import type { Knex } from 'knex';
+/**
+ * Runs `SELECT count(<countColumn>) FROM <table>` and returns the count
+ * as a plain JS number. Kept in this package instead of importing from
+ * `populate-ref-tables/test-utils/count-rows.ts` because a spec in populate-entities importing
+ * from populate-refs test-utils would establish a cross-directory test-utils
+ * dependency; the populate-refs copy has the same shape but its default
+ * countColumn is scoped to that phase's tables. Duplication is minimal
+ * and keeps each phase's test scaffolding self-contained.
+ * @param db - Knex instance (typically a spec-local in-memory DB).
+ * @param table - Table name.
+ * @param countColumn - Column to count; defaults to `'*'` so this helper
+ *   works uniformly across `content_items` / `page_meta` /
+ *   `resource_ref_edges` (some of which do not have an `id` column).
+ * @returns Row count.
+ */
+export declare function countRows(db: Knex, table: string, countColumn?: string): Promise<number>;

package/lib/archive/populate-entity-tables/test-utils/count-rows.js

@@ -0,0 +1,20 @@
+/**
+ * Runs `SELECT count(<countColumn>) FROM <table>` and returns the count
+ * as a plain JS number. Kept in this package instead of importing from
+ * `populate-ref-tables/test-utils/count-rows.ts` because a spec in populate-entities importing
+ * from populate-refs test-utils would establish a cross-directory test-utils
+ * dependency; the populate-refs copy has the same shape but its default
+ * countColumn is scoped to that phase's tables. Duplication is minimal
+ * and keeps each phase's test scaffolding self-contained.
+ * @param db - Knex instance (typically a spec-local in-memory DB).
+ * @param table - Table name.
+ * @param countColumn - Column to count; defaults to `'*'` so this helper
+ *   works uniformly across `content_items` / `page_meta` /
+ *   `resource_ref_edges` (some of which do not have an `id` column).
+ * @returns Row count.
+ */
+export async function countRows(db, table, countColumn = '*') {
+    const rows = await db(table).count({ n: countColumn });
+    const first = rows[0];
+    return Number(first.n);
+}

package/lib/archive/populate-entity-tables/test-utils/seed-content-items.d.ts

@@ -0,0 +1,25 @@
+import type knex from 'knex';
+/**
+ * Inserts one row into each of `pages`, `url_refs`, and `content_items` per
+ * requested id so downstream inserts against tables that FK to
+ * `content_items(id)` (`anchor_edges`, `image_items`, `page_meta`, …) satisfy
+ * their foreign key without every spec re-authoring the same 20 lines of
+ * seed logic.
+ *
+ * Shared by 0.13 check specs (`check-anchor-edges-count.spec.ts`,
+ * `check-anchor-edges-sum.spec.ts`, `check-image-items-count.spec.ts`,
+ * `verify-migration.spec.ts`) so a schema change to
+ * pages/url_refs/content_items only requires editing one file. Kept in
+ * `populate-entity-tables/test-utils/` alongside {@link setup-entities-db.ts} because that
+ * module already provisions the tables this helper writes to.
+ *
+ * Rows are inserted with defaults suitable for count / structural
+ * invariants: `scraped=1`, `isTarget=1`, `isExternal=0`, `source='crawled'`,
+ * URL `https://example.com/<id>` and matching `url_refs` row on the same id
+ * so `url_id` binds 1:1 with `pages.id` (helpful for URL round-trip specs).
+ * @param db - Knex handle from {@link setupMigrationDb}.
+ * @param ids - Content-item ids to create. Ordering does not matter but the
+ *   helper does not deduplicate the input list — callers should pass
+ *   distinct ids.
+ */
+export declare function seedContentItems(db: ReturnType<typeof knex>, ids: readonly number[]): Promise<void>;

package/lib/archive/populate-entity-tables/test-utils/seed-content-items.js

@@ -0,0 +1,42 @@
+/**
+ * Inserts one row into each of `pages`, `url_refs`, and `content_items` per
+ * requested id so downstream inserts against tables that FK to
+ * `content_items(id)` (`anchor_edges`, `image_items`, `page_meta`, …) satisfy
+ * their foreign key without every spec re-authoring the same 20 lines of
+ * seed logic.
+ *
+ * Shared by 0.13 check specs (`check-anchor-edges-count.spec.ts`,
+ * `check-anchor-edges-sum.spec.ts`, `check-image-items-count.spec.ts`,
+ * `verify-migration.spec.ts`) so a schema change to
+ * pages/url_refs/content_items only requires editing one file. Kept in
+ * `populate-entity-tables/test-utils/` alongside {@link setup-entities-db.ts} because that
+ * module already provisions the tables this helper writes to.
+ *
+ * Rows are inserted with defaults suitable for count / structural
+ * invariants: `scraped=1`, `isTarget=1`, `isExternal=0`, `source='crawled'`,
+ * URL `https://example.com/<id>` and matching `url_refs` row on the same id
+ * so `url_id` binds 1:1 with `pages.id` (helpful for URL round-trip specs).
+ * @param db - Knex handle from {@link setupMigrationDb}.
+ * @param ids - Content-item ids to create. Ordering does not matter but the
+ *   helper does not deduplicate the input list — callers should pass
+ *   distinct ids.
+ */
+export async function seedContentItems(db, ids) {
+    for (const id of ids) {
+        await db('pages').insert({
+            id,
+            url: `https://example.com/${id}`,
+            scraped: 1,
+            isTarget: 1,
+        });
+        await db('url_refs').insert({ id, url: `https://example.com/${id}` });
+        await db('content_items').insert({
+            id,
+            url_id: id,
+            is_external: 0,
+            scraped: 1,
+            is_target: 1,
+            source: 'crawled',
+        });
+    }
+}