@nitpicker/archive 0.0.0-stage → 0.25.0

package/lib/populate-entity-tables/populate-resource-ref-edges.d.ts

@@ -0,0 +1,31 @@
+import type { Knex } from 'knex';
+/**
+ * Populates `resource_ref_edges` from `resources-referrers`
+ * (issue #193).
+ *
+ * The transformation is a **direct SQL `INSERT ... SELECT`** — no ref
+ * lookups, no per-row JS work. Legacy `resources-referrers` already
+ * enforces `UNIQUE(resourceId, pageId)` (see `init-schema.ts`), so every
+ * source row maps 1:1 to a distinct `resource_ref_edges` PK; every
+ * `count` starts at `1` because the legacy shape observed each
+ * (resource, page) pair exactly once.
+ *
+ * `INSERT OR IGNORE` on the `(resource_id, page_id)` PK is redundant
+ * with the legacy unique constraint on the source but present anyway to
+ * make partial-failure re-runs safe.
+ *
+ * The PKs on both `resource_items.id` and `content_items.id` were
+ * preserved from `resources.id` / `pages.id`, so no per-row translation
+ * is required — the FKs `resource_items(id)` and `content_items(id)`
+ * declared in `resource_ref_edges` are already satisfied for every
+ * source row by `populateContentItems` / `populateResourceItems`, which
+ * run before this function in `populateEntityTables`.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateResourceItems(trx);
+ *   await populateContentItems(trx);
+ *   await populateResourceRefEdges(trx);
+ * });
+ */
+export declare function populateResourceRefEdges(trx: Knex): Promise<void>;

package/lib/populate-entity-tables/populate-resource-ref-edges.js

@@ -0,0 +1,33 @@
+/**
+ * Populates `resource_ref_edges` from `resources-referrers`
+ * (issue #193).
+ *
+ * The transformation is a **direct SQL `INSERT ... SELECT`** — no ref
+ * lookups, no per-row JS work. Legacy `resources-referrers` already
+ * enforces `UNIQUE(resourceId, pageId)` (see `init-schema.ts`), so every
+ * source row maps 1:1 to a distinct `resource_ref_edges` PK; every
+ * `count` starts at `1` because the legacy shape observed each
+ * (resource, page) pair exactly once.
+ *
+ * `INSERT OR IGNORE` on the `(resource_id, page_id)` PK is redundant
+ * with the legacy unique constraint on the source but present anyway to
+ * make partial-failure re-runs safe.
+ *
+ * The PKs on both `resource_items.id` and `content_items.id` were
+ * preserved from `resources.id` / `pages.id`, so no per-row translation
+ * is required — the FKs `resource_items(id)` and `content_items(id)`
+ * declared in `resource_ref_edges` are already satisfied for every
+ * source row by `populateContentItems` / `populateResourceItems`, which
+ * run before this function in `populateEntityTables`.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateResourceItems(trx);
+ *   await populateContentItems(trx);
+ *   await populateResourceRefEdges(trx);
+ * });
+ */
+export async function populateResourceRefEdges(trx) {
+    await trx.raw(`INSERT OR IGNORE INTO resource_ref_edges (resource_id, page_id, count)
+		 SELECT resourceId, pageId, 1 FROM "resources-referrers"`);
+}

package/lib/populate-entity-tables/resolve-blob-refs.d.ts

@@ -0,0 +1,31 @@
+import type { BlobRefIdMap } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Batch-resolves `blob_refs.id` for a set of large data-URI values
+ * (issue #193).
+ *
+ * The routing rule matches {@link ../populate-ref-tables/populate-blob-refs.ts}:
+ * only `data:` URIs longer than {@link DATA_URI_URL_REFS_LIMIT} bytes
+ * land in `blob_refs`; smaller data URIs (and every regular URL) live
+ * in `url_refs` instead. Callers pass URL-column values verbatim; this
+ * function filters out anything that does not match the routing rule
+ * before hashing.
+ *
+ * Malformed data URIs that fail {@link ../populate-ref-tables/decode-data-uri.ts}
+ * are skipped silently — `populateBlobRefs` also skipped them (with a
+ * warning), so no matching row exists to resolve.
+ *
+ * Duplicate values in `values` are deduped internally so an image whose
+ * `src` and `currentSrc` share the same data URI only counts once.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param values - Iterable of raw URL-column values that may or may not
+ *   be large data URIs.
+ * @returns Map keyed by the raw URI string; missing entries indicate
+ *   the value is not stored in `blob_refs` (either not a data URI at
+ *   all, below the size threshold, malformed, or absent from the
+ *   dictionary).
+ * @example
+ * const idMap = await resolveBlobRefs(trx, [longDataUri, 'https://…']);
+ * const blobId = idMap.get(longDataUri); // number | undefined
+ */
+export declare function resolveBlobRefs(trx: Knex, values: Iterable<string>): Promise<BlobRefIdMap>;

package/lib/populate-entity-tables/resolve-blob-refs.js

@@ -0,0 +1,100 @@
+import { computeContentHash } from '../populate-ref-tables/compute-content-hash.js';
+import { DATA_URI_URL_REFS_LIMIT } from '../populate-ref-tables/data-uri-url-refs-limit.js';
+import { decodeDataUri } from '../populate-ref-tables/decode-data-uri.js';
+/**
+ * Rows sent per `SELECT ... WHERE hash IN (?, ...)` chunk. `blob_refs`
+ * cardinality is tiny in the reference archive (≈ 429 rows) so 200 per
+ * chunk covers most realistic uses in a single query.
+ */
+const LOOKUP_CHUNK_SIZE = 200;
+/**
+ * Batch-resolves `blob_refs.id` for a set of large data-URI values
+ * (issue #193).
+ *
+ * The routing rule matches {@link ../populate-ref-tables/populate-blob-refs.ts}:
+ * only `data:` URIs longer than {@link DATA_URI_URL_REFS_LIMIT} bytes
+ * land in `blob_refs`; smaller data URIs (and every regular URL) live
+ * in `url_refs` instead. Callers pass URL-column values verbatim; this
+ * function filters out anything that does not match the routing rule
+ * before hashing.
+ *
+ * Malformed data URIs that fail {@link ../populate-ref-tables/decode-data-uri.ts}
+ * are skipped silently — `populateBlobRefs` also skipped them (with a
+ * warning), so no matching row exists to resolve.
+ *
+ * Duplicate values in `values` are deduped internally so an image whose
+ * `src` and `currentSrc` share the same data URI only counts once.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param values - Iterable of raw URL-column values that may or may not
+ *   be large data URIs.
+ * @returns Map keyed by the raw URI string; missing entries indicate
+ *   the value is not stored in `blob_refs` (either not a data URI at
+ *   all, below the size threshold, malformed, or absent from the
+ *   dictionary).
+ * @example
+ * const idMap = await resolveBlobRefs(trx, [longDataUri, 'https://…']);
+ * const blobId = idMap.get(longDataUri); // number | undefined
+ */
+export async function resolveBlobRefs(trx, values) {
+    const distinct = new Set();
+    for (const value of values) {
+        if (typeof value !== 'string' || value === '') {
+            continue;
+        }
+        if (value.length <= DATA_URI_URL_REFS_LIMIT || !value.startsWith('data:')) {
+            continue;
+        }
+        distinct.add(value);
+    }
+    if (distinct.size === 0) {
+        return new Map();
+    }
+    const rawValues = [...distinct];
+    const hashByValue = new Map();
+    // `hashHexToValues` maps `hex(hash) → [raw URI strings that decoded to
+    // those bytes]`. Two distinct raw data-URI strings can decode to the
+    // same payload bytes (e.g. `data:image/png;base64,XXXX…` and
+    // `data:image/svg+xml;base64,XXXX…` when the underlying base64 is
+    // identical). `populateBlobRefs` dedups them into one row keyed by the
+    // payload hash, so both raw URIs must resolve to that shared id — a
+    // last-wins Map keyed by hash would drop the earlier variant, and
+    // downstream `image_items` writes would lose the blob reference.
+    const hashHexToValues = new Map();
+    for (const value of rawValues) {
+        const decoded = decodeDataUri(value);
+        if (decoded === null) {
+            continue;
+        }
+        const hash = computeContentHash(decoded.bytes);
+        hashByValue.set(value, hash);
+        const hex = hash.toString('hex');
+        const bucket = hashHexToValues.get(hex);
+        if (bucket === undefined) {
+            hashHexToValues.set(hex, [value]);
+        }
+        else {
+            bucket.push(value);
+        }
+    }
+    if (hashByValue.size === 0) {
+        return new Map();
+    }
+    const hashes = [...hashByValue.values()];
+    const result = new Map();
+    for (let index = 0; index < hashes.length; index += LOOKUP_CHUNK_SIZE) {
+        const chunkHashes = hashes.slice(index, index + LOOKUP_CHUNK_SIZE);
+        const rows = await trx('blob_refs')
+            .select('id', 'hash')
+            .whereIn('hash', chunkHashes);
+        for (const row of rows) {
+            const hex = Buffer.from(row.hash).toString('hex');
+            const values = hashHexToValues.get(hex);
+            if (values !== undefined) {
+                for (const value of values) {
+                    result.set(value, row.id);
+                }
+            }
+        }
+    }
+    return result;
+}

package/lib/populate-entity-tables/resolve-content-type-refs.d.ts

@@ -0,0 +1,22 @@
+import type { ContentTypeRefIdMap } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Loads the entire `content_type_refs` dictionary into an in-process map
+ * keyed by the raw wire value (issue #193).
+ *
+ * Content-type cardinality is small in practice — the reference archive
+ * carries ≈ 400 distinct raw values across `pages` + `resources`, so
+ * loading everything into a `Map` costs ≈ 40 KB and eliminates every
+ * per-chunk round-trip. This is the same trade-off {@link
+ * ../populate-ref-tables/populate-header-tables.ts} makes for `header_name_refs`.
+ *
+ * Callers should invoke this once at the start of a 0.13 populate
+ * step (e.g. `populateContentItems`, `populateResourceItems`) and reuse
+ * the map for every chunk.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @returns Map keyed by `content_type_refs.raw`.
+ * @example
+ * const contentTypeIds = await loadContentTypeRefs(trx);
+ * const id = contentTypeIds.get('text/html; charset=utf-8'); // number | undefined
+ */
+export declare function loadContentTypeRefs(trx: Knex): Promise<ContentTypeRefIdMap>;

package/lib/populate-entity-tables/resolve-content-type-refs.js

@@ -0,0 +1,27 @@
+/**
+ * Loads the entire `content_type_refs` dictionary into an in-process map
+ * keyed by the raw wire value (issue #193).
+ *
+ * Content-type cardinality is small in practice — the reference archive
+ * carries ≈ 400 distinct raw values across `pages` + `resources`, so
+ * loading everything into a `Map` costs ≈ 40 KB and eliminates every
+ * per-chunk round-trip. This is the same trade-off {@link
+ * ../populate-ref-tables/populate-header-tables.ts} makes for `header_name_refs`.
+ *
+ * Callers should invoke this once at the start of a 0.13 populate
+ * step (e.g. `populateContentItems`, `populateResourceItems`) and reuse
+ * the map for every chunk.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @returns Map keyed by `content_type_refs.raw`.
+ * @example
+ * const contentTypeIds = await loadContentTypeRefs(trx);
+ * const id = contentTypeIds.get('text/html; charset=utf-8'); // number | undefined
+ */
+export async function loadContentTypeRefs(trx) {
+    const rows = await trx('content_type_refs').select('id', 'raw');
+    const map = new Map();
+    for (const row of rows) {
+        map.set(row.raw, row.id);
+    }
+    return map;
+}

package/lib/populate-entity-tables/resolve-header-sets.d.ts

@@ -0,0 +1,49 @@
+import type { HeaderSetIdMap } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Batch-resolves `header_sets.id` for a set of raw `responseHeaders`
+ * JSON strings (issue #193).
+ *
+ * The lookup is a two-stage cascade:
+ *
+ * 1. **Primary lookup** by `header_sets.raw_json_hash` — SHA-256 of the
+ *    raw JSON string exactly as stored on the source row. This hits when
+ *    the row's JSON key ordering matches whichever ordering populated
+ *    `header_sets` first for that stable-set equivalence class (see
+ *    {@link ../populate-ref-tables/populate-header-tables.ts}).
+ * 2. **Fallback lookup** by `header_sets.raw_hash` for the misses — SHA-
+ *    256 of the sorted `name=value` pairs. `populate-header-tables.ts`
+ *    stores exactly one `raw_json_hash` per `raw_hash` equivalence class
+ *    (the first JSON variant encountered), so a second variant with
+ *    identical decoded content but different key ordering never had its
+ *    `raw_json_hash` persisted — but its `raw_hash` still points at the
+ *    shared `header_sets` row. Without the fallback these rows would
+ *    silently get `header_set_id = NULL` on `content_items` /
+ *    `resource_items` and the count-based acceptance check would still
+ *    pass; the fallback closes that gap.
+ *
+ * Hashes are computed in JS because SQLite has no built-in BLAKE3 /
+ * SHA-256 (see {@link ../populate-ref-tables/compute-content-hash.ts} for the
+ * algorithm choice). The `raw_hash` fallback re-decomposes each miss via
+ * {@link ../populate-ref-tables/decompose-header-set.ts} so the canonicalisation
+ * matches the one `populateHeaderTables` used at insert time.
+ *
+ * Empty / null values and the sentinel `'{}'` / `'null'` strings are
+ * skipped — those responses were decomposed to `null` by
+ * `decomposeHeaderSet` and never produced a `header_sets` row, so the
+ * caller writes `header_set_id = null` for them.
+ *
+ * Duplicate JSON strings are deduped internally so a page and a
+ * resource sharing the same raw responseHeaders JSON only contribute
+ * one lookup slot.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param rawJsonStrings - Iterable of raw responseHeaders JSON strings.
+ * @returns Map keyed by the raw JSON string; missing entries indicate
+ *   no matching `header_sets` row exists (parse failure or empty set).
+ * @example
+ * const idMap = await resolveHeaderSets(trx, [
+ *   '{"content-type":"text/html"}',
+ * ]);
+ * const setId = idMap.get('{"content-type":"text/html"}'); // number | undefined
+ */
+export declare function resolveHeaderSets(trx: Knex, rawJsonStrings: Iterable<string>): Promise<HeaderSetIdMap>;

package/lib/populate-entity-tables/resolve-header-sets.js

@@ -0,0 +1,122 @@
+import { computeContentHash } from '../populate-ref-tables/compute-content-hash.js';
+import { decomposeHeaderSet } from '../populate-ref-tables/decompose-header-set.js';
+/**
+ * Rows sent per `SELECT ... WHERE raw_json_hash IN (?, ...)` chunk. Same
+ * rationale as {@link ./resolve-url-refs.ts}: 500 rows × 1 param each is
+ * well under the SQLite variable limit and gives good round-trip
+ * amortisation.
+ */
+const LOOKUP_CHUNK_SIZE = 500;
+/**
+ * Batch-resolves `header_sets.id` for a set of raw `responseHeaders`
+ * JSON strings (issue #193).
+ *
+ * The lookup is a two-stage cascade:
+ *
+ * 1. **Primary lookup** by `header_sets.raw_json_hash` — SHA-256 of the
+ *    raw JSON string exactly as stored on the source row. This hits when
+ *    the row's JSON key ordering matches whichever ordering populated
+ *    `header_sets` first for that stable-set equivalence class (see
+ *    {@link ../populate-ref-tables/populate-header-tables.ts}).
+ * 2. **Fallback lookup** by `header_sets.raw_hash` for the misses — SHA-
+ *    256 of the sorted `name=value` pairs. `populate-header-tables.ts`
+ *    stores exactly one `raw_json_hash` per `raw_hash` equivalence class
+ *    (the first JSON variant encountered), so a second variant with
+ *    identical decoded content but different key ordering never had its
+ *    `raw_json_hash` persisted — but its `raw_hash` still points at the
+ *    shared `header_sets` row. Without the fallback these rows would
+ *    silently get `header_set_id = NULL` on `content_items` /
+ *    `resource_items` and the count-based acceptance check would still
+ *    pass; the fallback closes that gap.
+ *
+ * Hashes are computed in JS because SQLite has no built-in BLAKE3 /
+ * SHA-256 (see {@link ../populate-ref-tables/compute-content-hash.ts} for the
+ * algorithm choice). The `raw_hash` fallback re-decomposes each miss via
+ * {@link ../populate-ref-tables/decompose-header-set.ts} so the canonicalisation
+ * matches the one `populateHeaderTables` used at insert time.
+ *
+ * Empty / null values and the sentinel `'{}'` / `'null'` strings are
+ * skipped — those responses were decomposed to `null` by
+ * `decomposeHeaderSet` and never produced a `header_sets` row, so the
+ * caller writes `header_set_id = null` for them.
+ *
+ * Duplicate JSON strings are deduped internally so a page and a
+ * resource sharing the same raw responseHeaders JSON only contribute
+ * one lookup slot.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param rawJsonStrings - Iterable of raw responseHeaders JSON strings.
+ * @returns Map keyed by the raw JSON string; missing entries indicate
+ *   no matching `header_sets` row exists (parse failure or empty set).
+ * @example
+ * const idMap = await resolveHeaderSets(trx, [
+ *   '{"content-type":"text/html"}',
+ * ]);
+ * const setId = idMap.get('{"content-type":"text/html"}'); // number | undefined
+ */
+export async function resolveHeaderSets(trx, rawJsonStrings) {
+    const distinct = new Set();
+    for (const raw of rawJsonStrings) {
+        if (typeof raw !== 'string' || raw === '' || raw === 'null' || raw === '{}') {
+            continue;
+        }
+        distinct.add(raw);
+    }
+    if (distinct.size === 0) {
+        return new Map();
+    }
+    const values = [...distinct];
+    const hashByValue = new Map();
+    const hashHexToValue = new Map();
+    for (const value of values) {
+        const hash = computeContentHash(value);
+        hashByValue.set(value, hash);
+        hashHexToValue.set(hash.toString('hex'), value);
+    }
+    const hashes = [...hashByValue.values()];
+    const result = new Map();
+    for (let index = 0; index < hashes.length; index += LOOKUP_CHUNK_SIZE) {
+        const chunkHashes = hashes.slice(index, index + LOOKUP_CHUNK_SIZE);
+        const rows = await trx('header_sets')
+            .select('id', 'raw_json_hash')
+            .whereIn('raw_json_hash', chunkHashes);
+        for (const row of rows) {
+            const hex = Buffer.from(row.raw_json_hash).toString('hex');
+            const value = hashHexToValue.get(hex);
+            if (value !== undefined) {
+                result.set(value, row.id);
+            }
+        }
+    }
+    // Fallback pass: any value we could not resolve by `raw_json_hash`
+    // might still map to an existing `header_sets` row via `raw_hash`
+    // (identical decoded content, different JSON key ordering).
+    const misses = values.filter((value) => !result.has(value));
+    if (misses.length === 0) {
+        return result;
+    }
+    const rawHashByValue = new Map();
+    const rawHashHexToValue = new Map();
+    for (const value of misses) {
+        const decomposed = decomposeHeaderSet(value);
+        if (decomposed === null) {
+            continue;
+        }
+        rawHashByValue.set(value, decomposed.rawHash);
+        rawHashHexToValue.set(decomposed.rawHash.toString('hex'), value);
+    }
+    const rawHashes = [...rawHashByValue.values()];
+    for (let index = 0; index < rawHashes.length; index += LOOKUP_CHUNK_SIZE) {
+        const chunkHashes = rawHashes.slice(index, index + LOOKUP_CHUNK_SIZE);
+        const rows = await trx('header_sets')
+            .select('id', 'raw_hash')
+            .whereIn('raw_hash', chunkHashes);
+        for (const row of rows) {
+            const hex = Buffer.from(row.raw_hash).toString('hex');
+            const value = rawHashHexToValue.get(hex);
+            if (value !== undefined) {
+                result.set(value, row.id);
+            }
+        }
+    }
+    return result;
+}

package/lib/populate-entity-tables/resolve-json-refs.d.ts

@@ -0,0 +1,25 @@
+import type { Knex } from 'knex';
+/**
+ * Batch-resolves `json_refs.id` for a set of raw `meta_extras` JSON
+ * strings (issue #193).
+ *
+ * The lookup goes through `json_refs.hash`, the SHA-256 hash of the raw
+ * JSON string stored by `populateJsonRefs` (see
+ * {@link ../populate-ref-tables/populate-json-refs.ts}). Hashes are computed in JS
+ * for the same reason as {@link ./resolve-header-sets.ts}: SQLite has no
+ * built-in hash function.
+ *
+ * Empty / null strings are skipped — those pages had no `meta_extras`
+ * and never produced a `json_refs` row, so the caller writes
+ * `meta_extras_json_id = null` for them.
+ *
+ * Duplicate strings in `rawJsonStrings` are deduped internally.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param rawJsonStrings - Iterable of raw `meta_extras` JSON strings.
+ * @returns Map keyed by the raw JSON string; missing entries indicate
+ *   no matching `json_refs` row exists.
+ * @example
+ * const idMap = await resolveJsonRefs(trx, ['{"customField":"x"}']);
+ * const jsonId = idMap.get('{"customField":"x"}'); // number | undefined
+ */
+export declare function resolveJsonRefs(trx: Knex, rawJsonStrings: Iterable<string>): Promise<ReadonlyMap<string, number>>;

package/lib/populate-entity-tables/resolve-json-refs.js

@@ -0,0 +1,67 @@
+import { computeContentHash } from '../populate-ref-tables/compute-content-hash.js';
+/**
+ * Rows sent per `SELECT ... WHERE hash IN (?, ...)` chunk. `json_refs`
+ * cardinality is bounded by the count of distinct `pages.meta_extras`
+ * JSON strings — typically low thousands on a reference archive — so
+ * 500 per chunk keeps round-trips low without straining the SQLite
+ * variable limit.
+ */
+const LOOKUP_CHUNK_SIZE = 500;
+/**
+ * Batch-resolves `json_refs.id` for a set of raw `meta_extras` JSON
+ * strings (issue #193).
+ *
+ * The lookup goes through `json_refs.hash`, the SHA-256 hash of the raw
+ * JSON string stored by `populateJsonRefs` (see
+ * {@link ../populate-ref-tables/populate-json-refs.ts}). Hashes are computed in JS
+ * for the same reason as {@link ./resolve-header-sets.ts}: SQLite has no
+ * built-in hash function.
+ *
+ * Empty / null strings are skipped — those pages had no `meta_extras`
+ * and never produced a `json_refs` row, so the caller writes
+ * `meta_extras_json_id = null` for them.
+ *
+ * Duplicate strings in `rawJsonStrings` are deduped internally.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param rawJsonStrings - Iterable of raw `meta_extras` JSON strings.
+ * @returns Map keyed by the raw JSON string; missing entries indicate
+ *   no matching `json_refs` row exists.
+ * @example
+ * const idMap = await resolveJsonRefs(trx, ['{"customField":"x"}']);
+ * const jsonId = idMap.get('{"customField":"x"}'); // number | undefined
+ */
+export async function resolveJsonRefs(trx, rawJsonStrings) {
+    const distinct = new Set();
+    for (const raw of rawJsonStrings) {
+        if (typeof raw === 'string' && raw !== '') {
+            distinct.add(raw);
+        }
+    }
+    if (distinct.size === 0) {
+        return new Map();
+    }
+    const values = [...distinct];
+    const hashByValue = new Map();
+    const hashHexToValue = new Map();
+    for (const value of values) {
+        const hash = computeContentHash(value);
+        hashByValue.set(value, hash);
+        hashHexToValue.set(hash.toString('hex'), value);
+    }
+    const hashes = [...hashByValue.values()];
+    const result = new Map();
+    for (let index = 0; index < hashes.length; index += LOOKUP_CHUNK_SIZE) {
+        const chunkHashes = hashes.slice(index, index + LOOKUP_CHUNK_SIZE);
+        const rows = await trx('json_refs')
+            .select('id', 'hash')
+            .whereIn('hash', chunkHashes);
+        for (const row of rows) {
+            const hex = Buffer.from(row.hash).toString('hex');
+            const value = hashHexToValue.get(hex);
+            if (value !== undefined) {
+                result.set(value, row.id);
+            }
+        }
+    }
+    return result;
+}

package/lib/populate-entity-tables/resolve-text-refs.d.ts

@@ -0,0 +1,30 @@
+import type { TextRefIdMap } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Batch-resolves `text_refs.id` for a set of raw text strings (issue #193).
+ *
+ * The `text_refs` UNIQUE constraint is on `(hash, text)` so lookups
+ * prefix-seek on `hash`. Each caller-supplied text is hashed in JS via
+ * {@link ../populate-ref-tables/compute-content-hash.ts} (32-byte SHA-256, matching
+ * how `populateTextRefs` inserted the rows) before the
+ * SQL query; providing the hash lets SQLite's index seek in O(log n)
+ * without a full table scan.
+ *
+ * The `(hash, text)` UNIQUE composite makes the trailing text column
+ * theoretically necessary to disambiguate the astronomically improbable
+ * hash collision — we narrow on both by including `text` in the WHERE
+ * clause, then filter the map keys by the exact text on read-back.
+ *
+ * Empty / null texts are ignored. Duplicate strings in `texts` are
+ * deduped internally.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param texts - Iterable of raw text strings to resolve.
+ * @returns Map keyed by the raw text; missing entries indicate the text
+ *   is not present in `text_refs` (which should not happen after
+ *   `populateTextRefs` completes but does happen mid-migration when a caller
+ *   passes texts that were never inserted).
+ * @example
+ * const idMap = await resolveTextRefs(trx, ['My Page Title', 'Alt text']);
+ * const titleId = idMap.get('My Page Title'); // number | undefined
+ */
+export declare function resolveTextRefs(trx: Knex, texts: Iterable<string>): Promise<TextRefIdMap>;

package/lib/populate-entity-tables/resolve-text-refs.js

@@ -0,0 +1,61 @@
+import { computeContentHash } from '../populate-ref-tables/compute-content-hash.js';
+/**
+ * Rows sent per `SELECT ... WHERE hash IN (?, ...)` chunk. Same rationale
+ * as {@link ./resolve-url-refs.ts}: 800 rows keeps the parameter count
+ * well under the SQLite variable limit even in the worst case where every
+ * hash is bound as its own parameter.
+ */
+const LOOKUP_CHUNK_SIZE = 800;
+/**
+ * Batch-resolves `text_refs.id` for a set of raw text strings (issue #193).
+ *
+ * The `text_refs` UNIQUE constraint is on `(hash, text)` so lookups
+ * prefix-seek on `hash`. Each caller-supplied text is hashed in JS via
+ * {@link ../populate-ref-tables/compute-content-hash.ts} (32-byte SHA-256, matching
+ * how `populateTextRefs` inserted the rows) before the
+ * SQL query; providing the hash lets SQLite's index seek in O(log n)
+ * without a full table scan.
+ *
+ * The `(hash, text)` UNIQUE composite makes the trailing text column
+ * theoretically necessary to disambiguate the astronomically improbable
+ * hash collision — we narrow on both by including `text` in the WHERE
+ * clause, then filter the map keys by the exact text on read-back.
+ *
+ * Empty / null texts are ignored. Duplicate strings in `texts` are
+ * deduped internally.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param texts - Iterable of raw text strings to resolve.
+ * @returns Map keyed by the raw text; missing entries indicate the text
+ *   is not present in `text_refs` (which should not happen after
+ *   `populateTextRefs` completes but does happen mid-migration when a caller
+ *   passes texts that were never inserted).
+ * @example
+ * const idMap = await resolveTextRefs(trx, ['My Page Title', 'Alt text']);
+ * const titleId = idMap.get('My Page Title'); // number | undefined
+ */
+export async function resolveTextRefs(trx, texts) {
+    const distinct = new Set();
+    for (const text of texts) {
+        if (typeof text === 'string' && text !== '') {
+            distinct.add(text);
+        }
+    }
+    if (distinct.size === 0) {
+        return new Map();
+    }
+    const values = [...distinct];
+    const hashes = values.map((text) => computeContentHash(text));
+    const result = new Map();
+    for (let index = 0; index < values.length; index += LOOKUP_CHUNK_SIZE) {
+        const chunkValues = values.slice(index, index + LOOKUP_CHUNK_SIZE);
+        const chunkHashes = hashes.slice(index, index + LOOKUP_CHUNK_SIZE);
+        const rows = await trx('text_refs')
+            .select('id', 'hash', 'text')
+            .whereIn('hash', chunkHashes)
+            .whereIn('text', chunkValues);
+        for (const row of rows) {
+            result.set(row.text, row.id);
+        }
+    }
+    return result;
+}

package/lib/populate-entity-tables/resolve-url-or-blob-from-maps.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Routes one legacy URL-shaped value to either `url_refs` or `blob_refs`
+ * per the {@link isBlobRefValue} threshold rule, looking the id up in
+ * pre-resolved maps rather than upserting (the populate steps batch-resolve
+ * every distinct value across a whole chunk before this per-row lookup —
+ * see `resolveUrlRefs` / `resolveBlobRefs`). At most one of `url` / `blob`
+ * is non-null; both may be `null` when the value is null or fails to
+ * resolve (e.g. the `blob_refs` row is missing because the data URI was
+ * malformed and skipped by `populateBlobRefs`).
+ * @param value - Raw URL-shaped column value.
+ * @param urlIds - Map of URL string → `url_refs.id`.
+ * @param blobIds - Map of data-URI string → `blob_refs.id`.
+ * @returns `{ url, blob }` pair with at most one non-null field.
+ * @example
+ * const slot = resolveUrlOrBlobFromMaps(row.src, urlIds, blobIds);
+ * // slot.url set for a regular URL, slot.blob set for a large data: URI.
+ */
+export declare function resolveUrlOrBlobFromMaps(value: string | null, urlIds: ReadonlyMap<string, number>, blobIds: ReadonlyMap<string, number>): {
+    url: number | null;
+    blob: number | null;
+};

package/lib/populate-entity-tables/resolve-url-or-blob-from-maps.js

@@ -0,0 +1,27 @@
+import { isBlobRefValue } from './is-blob-ref-value.js';
+/**
+ * Routes one legacy URL-shaped value to either `url_refs` or `blob_refs`
+ * per the {@link isBlobRefValue} threshold rule, looking the id up in
+ * pre-resolved maps rather than upserting (the populate steps batch-resolve
+ * every distinct value across a whole chunk before this per-row lookup —
+ * see `resolveUrlRefs` / `resolveBlobRefs`). At most one of `url` / `blob`
+ * is non-null; both may be `null` when the value is null or fails to
+ * resolve (e.g. the `blob_refs` row is missing because the data URI was
+ * malformed and skipped by `populateBlobRefs`).
+ * @param value - Raw URL-shaped column value.
+ * @param urlIds - Map of URL string → `url_refs.id`.
+ * @param blobIds - Map of data-URI string → `blob_refs.id`.
+ * @returns `{ url, blob }` pair with at most one non-null field.
+ * @example
+ * const slot = resolveUrlOrBlobFromMaps(row.src, urlIds, blobIds);
+ * // slot.url set for a regular URL, slot.blob set for a large data: URI.
+ */
+export function resolveUrlOrBlobFromMaps(value, urlIds, blobIds) {
+    if (typeof value !== 'string' || value === '') {
+        return { url: null, blob: null };
+    }
+    if (isBlobRefValue(value)) {
+        return { url: null, blob: blobIds.get(value) ?? null };
+    }
+    return { url: urlIds.get(value) ?? null, blob: null };
+}