@nitpicker/crawler 0.11.0 → 0.13.0

package/lib/archive/populate-entity-tables/populate-page-meta.d.ts

@@ -0,0 +1,33 @@
+import type { ProgressCallback } from '../create-progress-reporter.js';
+import type { Knex } from 'knex';
+/**
+ * Populates `page_meta` from `pages WHERE scraped = 1` (issue #193).
+ *
+ * Pages that were never scraped contribute no meta rows — they have no rendered title,
+ * description, or JSON-LD to preserve, so filtering at the SELECT keeps
+ * the reader working set narrow and matches the migration acceptance check
+ * `count(*) FROM page_meta = count(*) FROM pages WHERE scraped = 1`.
+ *
+ * Per chunk:
+ *
+ * 1. **Keyset-paginate** `pages.id > cursor AND scraped = 1`.
+ * 2. **Batch-resolve** all text / URL / JSON ref ids for every value in
+ *    the chunk — three resolvers, one round-trip per ref table.
+ * 3. **Bulk INSERT** with `page_id = pages.id`. `page_id` is both PK and
+ *    FK on `content_items(id)`; `populateContentItems` must have already
+ *    inserted the corresponding `content_items` row for the FK to be
+ *    valid at COMMIT time.
+ *
+ * `INSERT OR IGNORE` on the `page_id` PK makes the step idempotent so
+ * partial-failure re-runs are safe.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of `pages WHERE scraped = 1` scanned); see
+ *   {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateContentItems(trx);
+ *   await populatePageMeta(trx);
+ * });
+ */
+export declare function populatePageMeta(trx: Knex, onProgress?: ProgressCallback): Promise<void>;

package/lib/archive/populate-entity-tables/populate-page-meta.js

@@ -0,0 +1,267 @@
+import { createProgressReporter } from '../create-progress-reporter.js';
+import { PAGE_META_COLUMN_MAPS } from '../page-meta-column-maps.js';
+import { DATA_URI_URL_REFS_LIMIT } from '../populate-ref-tables/data-uri-url-refs-limit.js';
+import { resolveJsonRefs } from './resolve-json-refs.js';
+import { resolveTextRefs } from './resolve-text-refs.js';
+import { resolveUrlRefs } from './resolve-url-refs.js';
+/**
+ * Rows scanned per keyset-paginated `SELECT` chunk against
+ * `pages WHERE scraped = 1`. The row width here is dominated by long
+ * text columns (`meta_extras` JSON, `description`, `twitter_description`)
+ * so 400 rows caps peak chunk memory at ≈ 40 MB on the reference
+ * archive.
+ */
+const READ_CHUNK_SIZE = 400;
+/**
+ * Rows sent per `INSERT INTO page_meta ... VALUES (...)` statement. Each
+ * row binds 47 params (see the column list below), so 200 rows =
+ * 9 400 params — safely under SQLite's default variable limit.
+ */
+const INSERT_CHUNK_SIZE = 200;
+/**
+ * Populates `page_meta` from `pages WHERE scraped = 1` (issue #193).
+ *
+ * Pages that were never scraped contribute no meta rows — they have no rendered title,
+ * description, or JSON-LD to preserve, so filtering at the SELECT keeps
+ * the reader working set narrow and matches the migration acceptance check
+ * `count(*) FROM page_meta = count(*) FROM pages WHERE scraped = 1`.
+ *
+ * Per chunk:
+ *
+ * 1. **Keyset-paginate** `pages.id > cursor AND scraped = 1`.
+ * 2. **Batch-resolve** all text / URL / JSON ref ids for every value in
+ *    the chunk — three resolvers, one round-trip per ref table.
+ * 3. **Bulk INSERT** with `page_id = pages.id`. `page_id` is both PK and
+ *    FK on `content_items(id)`; `populateContentItems` must have already
+ *    inserted the corresponding `content_items` row for the FK to be
+ *    valid at COMMIT time.
+ *
+ * `INSERT OR IGNORE` on the `page_id` PK makes the step idempotent so
+ * partial-failure re-runs are safe.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of `pages WHERE scraped = 1` scanned); see
+ *   {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateContentItems(trx);
+ *   await populatePageMeta(trx);
+ * });
+ */
+export async function populatePageMeta(trx, onProgress) {
+    const textColumns = PAGE_META_COLUMN_MAPS.text.map((entry) => entry.source);
+    const urlColumns = PAGE_META_COLUMN_MAPS.url.map((entry) => entry.source);
+    const plainColumns = [
+        'id',
+        'lang',
+        'dir',
+        'charset',
+        'baseHref',
+        'viewport_raw',
+        'themeColor',
+        'applicationName',
+        'author',
+        'generator',
+        'publisher',
+        'robots_noindex',
+        'robots_nofollow',
+        'robots_noarchive',
+        'robots_noimageindex',
+        'googlebot',
+        'og_type',
+        'og_site_name',
+        'og_image_alt',
+        'og_image_width',
+        'og_image_height',
+        'og_locale',
+        'og_article_published_time',
+        'og_article_modified_time',
+        'twitter_card',
+        'twitter_site',
+        'twitter_creator',
+        'fb_app_id',
+        'verification_google',
+        'formatDetection_telephone',
+        'tag_count',
+        'jsonld_count',
+        'tags_providers_csv',
+        'meta_extras',
+    ];
+    const countRows = await trx('pages').where('scraped', true).count({ n: '*' });
+    const total = Number(countRows[0]?.n ?? 0);
+    const report = createProgressReporter('page_meta (pages)', total, onProgress);
+    let processed = 0;
+    let cursor = 0;
+    while (true) {
+        const rows = await trx('pages')
+            .select(...plainColumns, ...textColumns, ...urlColumns)
+            .where('id', '>', cursor)
+            .andWhere('scraped', true)
+            .orderBy('id', 'asc')
+            .limit(READ_CHUNK_SIZE);
+        if (rows.length === 0) {
+            break;
+        }
+        cursor = rows.at(-1).id;
+        processed += rows.length;
+        report(processed);
+        const texts = new Set();
+        const urls = new Set();
+        const jsonStrings = new Set();
+        for (const row of rows) {
+            for (const column of textColumns) {
+                const value = row[column];
+                if (typeof value === 'string' && value !== '') {
+                    texts.add(value);
+                }
+            }
+            for (const column of urlColumns) {
+                const value = row[column];
+                if (typeof value === 'string' && value !== '' && !isLargeDataUri(value)) {
+                    urls.add(value);
+                }
+            }
+            const metaExtras = row.meta_extras;
+            if (typeof metaExtras === 'string' && metaExtras !== '') {
+                jsonStrings.add(metaExtras);
+            }
+        }
+        const textIds = await resolveTextRefs(trx, texts);
+        const urlIds = await resolveUrlRefs(trx, urls);
+        const jsonIds = await resolveJsonRefs(trx, jsonStrings);
+        const inserts = rows.map((row) => {
+            const rowId = row.id;
+            const insert = {
+                page_id: rowId,
+                lang: row.lang ?? null,
+                dir: row.dir ?? null,
+                charset: row.charset ?? null,
+                base_href: row.baseHref ?? null,
+                viewport_raw: row.viewport_raw ?? null,
+                theme_color: row.themeColor ?? null,
+                application_name: row.applicationName ?? null,
+                author: row.author ?? null,
+                generator: row.generator ?? null,
+                publisher: row.publisher ?? null,
+                robots_noindex: row.robots_noindex ?? null,
+                robots_nofollow: row.robots_nofollow ?? null,
+                robots_noarchive: row.robots_noarchive ?? null,
+                robots_noimageindex: row.robots_noimageindex ?? null,
+                googlebot: row.googlebot ?? null,
+                og_type: row.og_type ?? null,
+                og_site_name: row.og_site_name ?? null,
+                og_image_alt: row.og_image_alt ?? null,
+                og_image_width: row.og_image_width ?? null,
+                og_image_height: row.og_image_height ?? null,
+                og_locale: row.og_locale ?? null,
+                og_article_published_time: row.og_article_published_time ?? null,
+                og_article_modified_time: row.og_article_modified_time ?? null,
+                twitter_card: row.twitter_card ?? null,
+                twitter_site: row.twitter_site ?? null,
+                twitter_creator: row.twitter_creator ?? null,
+                fb_app_id: row.fb_app_id ?? null,
+                verification_google: row.verification_google ?? null,
+                format_detection_telephone: row.formatDetection_telephone ?? null,
+                tag_count: row.tag_count ?? null,
+                jsonld_count: row.jsonld_count ?? null,
+                tags_providers_csv: row.tags_providers_csv ?? null,
+                meta_extras_json_id: resolveOptionalJsonId(row.meta_extras, jsonIds, rowId),
+            };
+            for (const { source, target } of PAGE_META_COLUMN_MAPS.text) {
+                insert[target] = resolveOptionalTextId(row[source], textIds, rowId, source);
+            }
+            for (const { source, target } of PAGE_META_COLUMN_MAPS.url) {
+                insert[target] = resolveOptionalUrlId(row[source], urlIds, rowId, source);
+            }
+            return insert;
+        });
+        for (let index = 0; index < inserts.length; index += INSERT_CHUNK_SIZE) {
+            const chunk = inserts.slice(index, index + INSERT_CHUNK_SIZE);
+            await trx('page_meta').insert(chunk).onConflict('page_id').ignore();
+        }
+    }
+}
+/**
+ * Returns `true` when `value` is a `data:` URI larger than the routing
+ * threshold — i.e. one that the `url_refs` populate
+ * (`populate-ref-tables/populate-url-refs.ts`) sent to `blob_refs` instead
+ * of `url_refs`. `page_meta` deliberately has no `*_blob_id` companion
+ * columns for its URL slots, so we skip these values from
+ * the `url_refs` lookup rather than issue a doomed `WHERE url IN (?)`
+ * that would burn a chunk slot per multi-KB URI. The resulting
+ * `*_url_id` is null with a `console.warn` breadcrumb; the operator
+ * can then decide whether the archive needs a schema extension or the
+ * raw data warrants normalisation.
+ * @param value - Raw URL column value.
+ */
+function isLargeDataUri(value) {
+    return value.startsWith('data:') && value.length > DATA_URI_URL_REFS_LIMIT;
+}
+/**
+ * Resolves `text_refs.id` for a `page_meta.<name>_text_id` slot or
+ * throws when a non-empty source value has no matching `text_refs` row
+ * (parity with `populateContentItems`' `url_id` throw — silent NULL
+ * corruption would still let the count-based acceptance invariant pass
+ * while individual `page_meta` rows lose their meta text).
+ * @param value - Raw column value from the source `pages` row.
+ * @param textIds - `text → text_refs.id` map batch-resolved for the
+ *   current chunk (see {@link ./resolve-text-refs.ts}).
+ * @param pageId - Owning page id (for the error message).
+ * @param column - Source column name (for the error message).
+ */
+function resolveOptionalTextId(value, textIds, pageId, column) {
+    if (typeof value !== 'string' || value === '') {
+        return null;
+    }
+    const id = textIds.get(value);
+    if (id === undefined) {
+        throw new Error(`populatePageMeta: text_refs.id not resolved for page id=${pageId} column=${column} — populateTextRefs must run first`);
+    }
+    return id;
+}
+/**
+ * Resolves `url_refs.id` for a `page_meta.<name>_url_id` slot, throws
+ * on a missing lookup for a regular URL, and returns `null` (with a
+ * warning) for large data URIs that the `url_refs` populate routed to
+ * `blob_refs` — those cannot land in `page_meta` because the schema has
+ * no `*_blob_id` companion columns.
+ * @param value - Raw column value from the source `pages` row.
+ * @param urlIds - `url → url_refs.id` map batch-resolved for the
+ *   current chunk (see {@link ./resolve-url-refs.ts}).
+ * @param pageId - Owning page id (for the error message).
+ * @param column - Source column name (for the error message).
+ */
+function resolveOptionalUrlId(value, urlIds, pageId, column) {
+    if (typeof value !== 'string' || value === '') {
+        return null;
+    }
+    if (isLargeDataUri(value)) {
+        // eslint-disable-next-line no-console
+        console.warn(`populatePageMeta: dropping large data URI in page id=${pageId} column=${column} (page_meta has no *_blob_id companion for URL columns)`);
+        return null;
+    }
+    const id = urlIds.get(value);
+    if (id === undefined) {
+        throw new Error(`populatePageMeta: url_refs.id not resolved for page id=${pageId} column=${column} — populateUrlRefs must run first`);
+    }
+    return id;
+}
+/**
+ * Resolves `json_refs.id` for `page_meta.meta_extras_json_id` or throws
+ * when the source `meta_extras` string has no matching `json_refs` row
+ * (parity with the text/url branches above).
+ * @param value - Raw `meta_extras` value from the source `pages` row.
+ * @param jsonIds - `JSON string → json_refs.id` map batch-resolved for
+ *   the current chunk (see {@link ./resolve-json-refs.ts}).
+ * @param pageId - Owning page id (for the error message).
+ */
+function resolveOptionalJsonId(value, jsonIds, pageId) {
+    if (typeof value !== 'string' || value === '') {
+        return null;
+    }
+    const id = jsonIds.get(value);
+    if (id === undefined) {
+        throw new Error(`populatePageMeta: json_refs.id not resolved for page id=${pageId} column=meta_extras — populateJsonRefs must run first`);
+    }
+    return id;
+}

package/lib/archive/populate-entity-tables/populate-resource-items.d.ts

@@ -0,0 +1,22 @@
+import type { ProgressCallback } from '../create-progress-reporter.js';
+import type { Knex } from 'knex';
+/**
+ * Populates `resource_items` from `resources` (issue #193).
+ *
+ * Structurally analogous to {@link ./populate-content-items.ts}: for each
+ * `resources` chunk, batch-resolve `url_refs.id`, `content_type_refs.id`,
+ * and `header_sets.id`, then bulk-INSERT with explicit `id = resources.id`.
+ * Reusing the legacy PK preserves the `resources-referrers.resourceId`
+ * FK reference (which `populateResourceRefEdges` later carries over as
+ * `resource_ref_edges.resource_id`) without any per-row UPDATE.
+ *
+ * `INSERT OR IGNORE` on `id` makes the step idempotent.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of `resources` scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateResourceItems(trx);
+ * });
+ */
+export declare function populateResourceItems(trx: Knex, onProgress?: ProgressCallback): Promise<void>;

package/lib/archive/populate-entity-tables/populate-resource-items.js

@@ -0,0 +1,114 @@
+import { createProgressReporter } from '../create-progress-reporter.js';
+import { isBlobRefValue } from './is-blob-ref-value.js';
+import { resolveBlobRefs } from './resolve-blob-refs.js';
+import { loadContentTypeRefs } from './resolve-content-type-refs.js';
+import { resolveHeaderSets } from './resolve-header-sets.js';
+import { resolveUrlOrBlobFromMaps } from './resolve-url-or-blob-from-maps.js';
+import { resolveUrlRefs } from './resolve-url-refs.js';
+/**
+ * Rows scanned per keyset-paginated `SELECT` chunk against `resources`.
+ * `resources` rows are narrow (no meta columns) so 800 rows fits
+ * comfortably in a chunk without stressing memory. Bumping this higher
+ * would help throughput but risks pushing large `responseHeaders` JSON
+ * blobs past a healthy per-result-set size on archives with per-row
+ * multi-KB header payloads.
+ */
+const READ_CHUNK_SIZE = 800;
+/**
+ * Rows sent per `INSERT INTO resource_items ... VALUES (...)` statement.
+ * Each row binds 11 params (id + 10 columns), so 500 rows = 5 500 params
+ * — well under SQLite's default variable limit.
+ */
+const INSERT_CHUNK_SIZE = 500;
+/**
+ * Populates `resource_items` from `resources` (issue #193).
+ *
+ * Structurally analogous to {@link ./populate-content-items.ts}: for each
+ * `resources` chunk, batch-resolve `url_refs.id`, `content_type_refs.id`,
+ * and `header_sets.id`, then bulk-INSERT with explicit `id = resources.id`.
+ * Reusing the legacy PK preserves the `resources-referrers.resourceId`
+ * FK reference (which `populateResourceRefEdges` later carries over as
+ * `resource_ref_edges.resource_id`) without any per-row UPDATE.
+ *
+ * `INSERT OR IGNORE` on `id` makes the step idempotent.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of `resources` scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateResourceItems(trx);
+ * });
+ */
+export async function populateResourceItems(trx, onProgress) {
+    const contentTypeIds = await loadContentTypeRefs(trx);
+    const countRows = await trx('resources').count({ n: '*' });
+    const total = Number(countRows[0]?.n ?? 0);
+    const report = createProgressReporter('resource_items (resources)', total, onProgress);
+    let processed = 0;
+    let cursor = 0;
+    while (true) {
+        const rows = await trx('resources')
+            .select('id', 'url', 'isExternal', 'status', 'statusText', 'contentType', 'contentLength', 'responseHeaders', 'compress', 'cdn', 'source')
+            .where('id', '>', cursor)
+            .orderBy('id', 'asc')
+            .limit(READ_CHUNK_SIZE);
+        if (rows.length === 0) {
+            break;
+        }
+        cursor = rows.at(-1).id;
+        processed += rows.length;
+        report(processed);
+        const urls = new Set();
+        const dataUris = new Set();
+        const headerJsonStrings = new Set();
+        for (const row of rows) {
+            if (isBlobRefValue(row.url)) {
+                dataUris.add(row.url);
+            }
+            else {
+                urls.add(row.url);
+            }
+            if (typeof row.responseHeaders === 'string' && row.responseHeaders !== '') {
+                headerJsonStrings.add(row.responseHeaders);
+            }
+        }
+        const urlIds = await resolveUrlRefs(trx, urls);
+        const blobIds = await resolveBlobRefs(trx, dataUris);
+        const headerSetIds = await resolveHeaderSets(trx, headerJsonStrings);
+        const inserts = rows.map((row) => {
+            const slot = resolveUrlOrBlobFromMaps(row.url, urlIds, blobIds);
+            if (slot.url === null && slot.blob === null) {
+                throw new Error(`populateResourceItems: neither url_refs.id nor blob_refs.id resolved for resource id=${row.id} url=${row.url.slice(0, 80)} — populateUrlRefs/populateBlobRefs must run first`);
+            }
+            let contentTypeId = null;
+            if (row.contentType != null && row.contentType !== '') {
+                const resolved = contentTypeIds.get(row.contentType);
+                if (resolved === undefined) {
+                    throw new Error(`populateResourceItems: content_type_refs.id not resolved for resource id=${row.id} contentType=${row.contentType} — populateContentTypeRefs must run first`);
+                }
+                contentTypeId = resolved;
+            }
+            const headerSetId = typeof row.responseHeaders === 'string' && row.responseHeaders !== ''
+                ? (headerSetIds.get(row.responseHeaders) ?? null)
+                : null;
+            return {
+                id: row.id,
+                url_id: slot.url,
+                url_blob_id: slot.blob,
+                is_external: row.isExternal == null ? 0 : row.isExternal ? 1 : 0,
+                status: row.status ?? null,
+                status_text: row.statusText ?? null,
+                content_type_id: contentTypeId,
+                content_length: row.contentLength ?? null,
+                header_set_id: headerSetId,
+                compress: row.compress ?? null,
+                cdn: row.cdn ?? null,
+                source: row.source,
+            };
+        });
+        for (let index = 0; index < inserts.length; index += INSERT_CHUNK_SIZE) {
+            const chunk = inserts.slice(index, index + INSERT_CHUNK_SIZE);
+            await trx('resource_items').insert(chunk).onConflict('id').ignore();
+        }
+    }
+}

package/lib/archive/populate-entity-tables/populate-resource-ref-edges.d.ts

@@ -0,0 +1,31 @@
+import type { Knex } from 'knex';
+/**
+ * Populates `resource_ref_edges` from `resources-referrers`
+ * (issue #193).
+ *
+ * The transformation is a **direct SQL `INSERT ... SELECT`** — no ref
+ * lookups, no per-row JS work. Legacy `resources-referrers` already
+ * enforces `UNIQUE(resourceId, pageId)` (see `init-schema.ts`), so every
+ * source row maps 1:1 to a distinct `resource_ref_edges` PK; every
+ * `count` starts at `1` because the legacy shape observed each
+ * (resource, page) pair exactly once.
+ *
+ * `INSERT OR IGNORE` on the `(resource_id, page_id)` PK is redundant
+ * with the legacy unique constraint on the source but present anyway to
+ * make partial-failure re-runs safe.
+ *
+ * The PKs on both `resource_items.id` and `content_items.id` were
+ * preserved from `resources.id` / `pages.id`, so no per-row translation
+ * is required — the FKs `resource_items(id)` and `content_items(id)`
+ * declared in `resource_ref_edges` are already satisfied for every
+ * source row by `populateContentItems` / `populateResourceItems`, which
+ * run before this function in `populateEntityTables`.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateResourceItems(trx);
+ *   await populateContentItems(trx);
+ *   await populateResourceRefEdges(trx);
+ * });
+ */
+export declare function populateResourceRefEdges(trx: Knex): Promise<void>;

package/lib/archive/populate-entity-tables/populate-resource-ref-edges.js

@@ -0,0 +1,33 @@
+/**
+ * Populates `resource_ref_edges` from `resources-referrers`
+ * (issue #193).
+ *
+ * The transformation is a **direct SQL `INSERT ... SELECT`** — no ref
+ * lookups, no per-row JS work. Legacy `resources-referrers` already
+ * enforces `UNIQUE(resourceId, pageId)` (see `init-schema.ts`), so every
+ * source row maps 1:1 to a distinct `resource_ref_edges` PK; every
+ * `count` starts at `1` because the legacy shape observed each
+ * (resource, page) pair exactly once.
+ *
+ * `INSERT OR IGNORE` on the `(resource_id, page_id)` PK is redundant
+ * with the legacy unique constraint on the source but present anyway to
+ * make partial-failure re-runs safe.
+ *
+ * The PKs on both `resource_items.id` and `content_items.id` were
+ * preserved from `resources.id` / `pages.id`, so no per-row translation
+ * is required — the FKs `resource_items(id)` and `content_items(id)`
+ * declared in `resource_ref_edges` are already satisfied for every
+ * source row by `populateContentItems` / `populateResourceItems`, which
+ * run before this function in `populateEntityTables`.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateResourceItems(trx);
+ *   await populateContentItems(trx);
+ *   await populateResourceRefEdges(trx);
+ * });
+ */
+export async function populateResourceRefEdges(trx) {
+    await trx.raw(`INSERT OR IGNORE INTO resource_ref_edges (resource_id, page_id, count)
+		 SELECT resourceId, pageId, 1 FROM "resources-referrers"`);
+}

package/lib/archive/populate-entity-tables/resolve-blob-refs.d.ts

@@ -0,0 +1,31 @@
+import type { BlobRefIdMap } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Batch-resolves `blob_refs.id` for a set of large data-URI values
+ * (issue #193).
+ *
+ * The routing rule matches {@link ../populate-ref-tables/populate-blob-refs.ts}:
+ * only `data:` URIs longer than {@link DATA_URI_URL_REFS_LIMIT} bytes
+ * land in `blob_refs`; smaller data URIs (and every regular URL) live
+ * in `url_refs` instead. Callers pass URL-column values verbatim; this
+ * function filters out anything that does not match the routing rule
+ * before hashing.
+ *
+ * Malformed data URIs that fail {@link ../populate-ref-tables/decode-data-uri.ts}
+ * are skipped silently — `populateBlobRefs` also skipped them (with a
+ * warning), so no matching row exists to resolve.
+ *
+ * Duplicate values in `values` are deduped internally so an image whose
+ * `src` and `currentSrc` share the same data URI only counts once.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param values - Iterable of raw URL-column values that may or may not
+ *   be large data URIs.
+ * @returns Map keyed by the raw URI string; missing entries indicate
+ *   the value is not stored in `blob_refs` (either not a data URI at
+ *   all, below the size threshold, malformed, or absent from the
+ *   dictionary).
+ * @example
+ * const idMap = await resolveBlobRefs(trx, [longDataUri, 'https://…']);
+ * const blobId = idMap.get(longDataUri); // number | undefined
+ */
+export declare function resolveBlobRefs(trx: Knex, values: Iterable<string>): Promise<BlobRefIdMap>;

package/lib/archive/populate-entity-tables/resolve-blob-refs.js

@@ -0,0 +1,100 @@
+import { computeContentHash } from '../populate-ref-tables/compute-content-hash.js';
+import { DATA_URI_URL_REFS_LIMIT } from '../populate-ref-tables/data-uri-url-refs-limit.js';
+import { decodeDataUri } from '../populate-ref-tables/decode-data-uri.js';
+/**
+ * Rows sent per `SELECT ... WHERE hash IN (?, ...)` chunk. `blob_refs`
+ * cardinality is tiny in the reference archive (≈ 429 rows) so 200 per
+ * chunk covers most realistic uses in a single query.
+ */
+const LOOKUP_CHUNK_SIZE = 200;
+/**
+ * Batch-resolves `blob_refs.id` for a set of large data-URI values
+ * (issue #193).
+ *
+ * The routing rule matches {@link ../populate-ref-tables/populate-blob-refs.ts}:
+ * only `data:` URIs longer than {@link DATA_URI_URL_REFS_LIMIT} bytes
+ * land in `blob_refs`; smaller data URIs (and every regular URL) live
+ * in `url_refs` instead. Callers pass URL-column values verbatim; this
+ * function filters out anything that does not match the routing rule
+ * before hashing.
+ *
+ * Malformed data URIs that fail {@link ../populate-ref-tables/decode-data-uri.ts}
+ * are skipped silently — `populateBlobRefs` also skipped them (with a
+ * warning), so no matching row exists to resolve.
+ *
+ * Duplicate values in `values` are deduped internally so an image whose
+ * `src` and `currentSrc` share the same data URI only counts once.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param values - Iterable of raw URL-column values that may or may not
+ *   be large data URIs.
+ * @returns Map keyed by the raw URI string; missing entries indicate
+ *   the value is not stored in `blob_refs` (either not a data URI at
+ *   all, below the size threshold, malformed, or absent from the
+ *   dictionary).
+ * @example
+ * const idMap = await resolveBlobRefs(trx, [longDataUri, 'https://…']);
+ * const blobId = idMap.get(longDataUri); // number | undefined
+ */
+export async function resolveBlobRefs(trx, values) {
+    const distinct = new Set();
+    for (const value of values) {
+        if (typeof value !== 'string' || value === '') {
+            continue;
+        }
+        if (value.length <= DATA_URI_URL_REFS_LIMIT || !value.startsWith('data:')) {
+            continue;
+        }
+        distinct.add(value);
+    }
+    if (distinct.size === 0) {
+        return new Map();
+    }
+    const rawValues = [...distinct];
+    const hashByValue = new Map();
+    // `hashHexToValues` maps `hex(hash) → [raw URI strings that decoded to
+    // those bytes]`. Two distinct raw data-URI strings can decode to the
+    // same payload bytes (e.g. `data:image/png;base64,XXXX…` and
+    // `data:image/svg+xml;base64,XXXX…` when the underlying base64 is
+    // identical). `populateBlobRefs` dedups them into one row keyed by the
+    // payload hash, so both raw URIs must resolve to that shared id — a
+    // last-wins Map keyed by hash would drop the earlier variant, and
+    // downstream `image_items` writes would lose the blob reference.
+    const hashHexToValues = new Map();
+    for (const value of rawValues) {
+        const decoded = decodeDataUri(value);
+        if (decoded === null) {
+            continue;
+        }
+        const hash = computeContentHash(decoded.bytes);
+        hashByValue.set(value, hash);
+        const hex = hash.toString('hex');
+        const bucket = hashHexToValues.get(hex);
+        if (bucket === undefined) {
+            hashHexToValues.set(hex, [value]);
+        }
+        else {
+            bucket.push(value);
+        }
+    }
+    if (hashByValue.size === 0) {
+        return new Map();
+    }
+    const hashes = [...hashByValue.values()];
+    const result = new Map();
+    for (let index = 0; index < hashes.length; index += LOOKUP_CHUNK_SIZE) {
+        const chunkHashes = hashes.slice(index, index + LOOKUP_CHUNK_SIZE);
+        const rows = await trx('blob_refs')
+            .select('id', 'hash')
+            .whereIn('hash', chunkHashes);
+        for (const row of rows) {
+            const hex = Buffer.from(row.hash).toString('hex');
+            const values = hashHexToValues.get(hex);
+            if (values !== undefined) {
+                for (const value of values) {
+                    result.set(value, row.id);
+                }
+            }
+        }
+    }
+    return result;
+}

package/lib/archive/populate-entity-tables/resolve-content-type-refs.d.ts

@@ -0,0 +1,22 @@
+import type { ContentTypeRefIdMap } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Loads the entire `content_type_refs` dictionary into an in-process map
+ * keyed by the raw wire value (issue #193).
+ *
+ * Content-type cardinality is small in practice — the reference archive
+ * carries ≈ 400 distinct raw values across `pages` + `resources`, so
+ * loading everything into a `Map` costs ≈ 40 KB and eliminates every
+ * per-chunk round-trip. This is the same trade-off {@link
+ * ../populate-ref-tables/populate-header-tables.ts} makes for `header_name_refs`.
+ *
+ * Callers should invoke this once at the start of a 0.13 populate
+ * step (e.g. `populateContentItems`, `populateResourceItems`) and reuse
+ * the map for every chunk.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @returns Map keyed by `content_type_refs.raw`.
+ * @example
+ * const contentTypeIds = await loadContentTypeRefs(trx);
+ * const id = contentTypeIds.get('text/html; charset=utf-8'); // number | undefined
+ */
+export declare function loadContentTypeRefs(trx: Knex): Promise<ContentTypeRefIdMap>;