@nitpicker/crawler 0.11.0 → 0.13.0

package/lib/archive/populate-ref-tables/decompose-header-set.js

@@ -0,0 +1,157 @@
+import { computeContentHash } from './compute-content-hash.js';
+import { isVolatileHeader } from './header-stability.js';
+/**
+ * Parses one raw `responseHeaders` JSON string and produces the derived
+ * data shape required to insert into `header_sets`, `header_set_entries`,
+ * and `header_flags`.
+ *
+ * Behaviour:
+ *
+ * - `null` / `""` / `"null"` / `"{}"` / non-object JSON → returns `null`.
+ *   The caller sets `header_set_id = null` on the referring row.
+ * - Multiple values per header name (JSON arrays, e.g.
+ *   `{ "set-cookie": ["a=1", "b=2"] }`) become multiple
+ *   {@link HeaderEntry} rows with `occurrence` ordinals 1, 2, ... The
+ *   composite PK on `header_set_entries` `(header_set_id, name_id,
+ *   occurrence)` guarantees no truncation.
+ * - Names are lower-cased. Values are trimmed of whitespace only in the
+ *   sorted-hash construction; the raw value is preserved verbatim in
+ *   the emitted `HeaderEntry.value` (and hence in `header_value_refs`).
+ *   Trimming inside hash construction lets `content-type: text/html`
+ *   and `content-type:  text/html ` dedup to one stable set.
+ * - Sort order for hashing is `(name, occurrence)` in binary form —
+ *   deterministic regardless of the input JSON's insertion order or
+ *   how the JSON serializer arranged keys.
+ * @param rawJson - The raw JSON string exactly as stored in
+ *   `pages.responseHeaders` / `resources.responseHeaders`.
+ * @returns Decomposed shape, or `null` when the row has no meaningful
+ *   header set.
+ */
+export function decomposeHeaderSet(rawJson) {
+    if (rawJson == null || rawJson === '' || rawJson === 'null' || rawJson === '{}') {
+        return null;
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(rawJson);
+    }
+    catch {
+        return null;
+    }
+    if (parsed == null || typeof parsed !== 'object' || Array.isArray(parsed)) {
+        return null;
+    }
+    const entries = [];
+    // `occurrence` runs per (lower-cased name) across ALL sources for that
+    // name, so duplicate JSON keys that differ only in case
+    // (`{"Cookie":"a=1", "cookie":"b=2"}` — non-conforming but legal JSON,
+    // and the crawler stores whatever puppeteer hands it) don't collide on
+    // (name_id, occurrence). Without this counter the second entry would
+    // land at occurrence=1 too, violating the `header_set_entries` PK and
+    // silently getting dropped by INSERT OR IGNORE while `entry_count`
+    // still counted it.
+    const occurrenceByName = new Map();
+    for (const [rawName, rawValue] of Object.entries(parsed)) {
+        const name = rawName.toLowerCase();
+        const volatile = isVolatileHeader(name);
+        const nextOccurrence = () => {
+            const next = (occurrenceByName.get(name) ?? 0) + 1;
+            occurrenceByName.set(name, next);
+            return next;
+        };
+        if (Array.isArray(rawValue)) {
+            for (const one of rawValue) {
+                if (typeof one !== 'string') {
+                    continue;
+                }
+                entries.push({
+                    name,
+                    value: one,
+                    occurrence: nextOccurrence(),
+                    isVolatile: volatile,
+                });
+            }
+            continue;
+        }
+        if (typeof rawValue !== 'string') {
+            // parseResponseHeaders's stored shape is `Record<string, string |
+            // string[] | undefined>`, but hand-edited or non-conforming
+            // archives could smuggle in a nested object / number / boolean
+            // — coercing via `String()` would produce "[object Object]" and
+            // pollute `header_value_refs` with a garbage dictionary entry.
+            // Skipping is the conservative choice; the source row's raw
+            // JSON is still preserved via `raw_json_hash`.
+            continue;
+        }
+        entries.push({
+            name,
+            value: rawValue,
+            occurrence: nextOccurrence(),
+            isVolatile: volatile,
+        });
+    }
+    if (entries.length === 0) {
+        return null;
+    }
+    entries.sort((a, b) => {
+        if (a.name < b.name)
+            return -1;
+        if (a.name > b.name)
+            return 1;
+        return a.occurrence - b.occurrence;
+    });
+    // Canonicalize every entry exactly once, then partition the strings.
+    // Canonicalizing per hash pass instead (once in the all-entries pass,
+    // again in the stable / volatile passes) would serialize each entry
+    // twice, which becomes measurable on large archives where a single
+    // response can carry a couple of dozen entries.
+    const canonicalStrings = entries.map(canonicalize);
+    const stableCanonical = [];
+    const volatileCanonical = [];
+    let stableEntryCount = 0;
+    for (const [i, entry] of entries.entries()) {
+        if (entry.isVolatile) {
+            volatileCanonical.push(canonicalStrings[i]);
+        }
+        else {
+            stableCanonical.push(canonicalStrings[i]);
+            stableEntryCount += 1;
+        }
+    }
+    const rawJsonHash = computeContentHash(rawJson);
+    const rawHash = computeContentHash(canonicalStrings.join('\n'));
+    // A response with only volatile headers (tracker endpoints that return
+    // just `Date` / `Set-Cookie` / `Age`) hashes to `computeContentHash('')`
+    // — the canonical "empty stable profile" hash. All all-volatile
+    // responses cluster under this sentinel, which is the intended
+    // semantic (same empty stable profile = same cluster), and 0.13's
+    // DDL declares `stable_hash BLOB NOT NULL` so a `null` sentinel is not
+    // available even if we wanted one. Consumers that need to distinguish
+    // "empty stable set" from "populated stable set" should key off
+    // `stable_entry_count === 0` on the same row, not off the hash.
+    const stableHash = computeContentHash(stableCanonical.join('\n'));
+    const volatileHash = volatileCanonical.length === 0
+        ? null
+        : computeContentHash(volatileCanonical.join('\n'));
+    return {
+        rawJsonHash,
+        rawHash,
+        stableHash,
+        volatileHash,
+        entries,
+        entryCount: entries.length,
+        stableEntryCount,
+    };
+}
+/**
+ * Serializes one entry into the canonical `name/occurrence=value` form
+ * used inside the hash construction. The trailing occurrence keeps
+ * multi-value headers distinguishable while still letting single-value
+ * ones dedup naturally. Values are trimmed so surrounding whitespace
+ * does not fork the dedup key.
+ * @param entry - One decomposed header entry.
+ * @returns Canonical string joined into the hash pre-image.
+ */
+function canonicalize(entry) {
+    return `${entry.name}/${entry.occurrence}=${entry.value.trim()}`;
+}

package/lib/archive/populate-ref-tables/decompose-url.d.ts

@@ -0,0 +1,25 @@
+import type { DecomposedUrl } from './types.js';
+/**
+ * Extracts `scheme` / `host` / `port` / `path` / `query_hash` / `fragment`
+ * columns from a URL string for `url_refs` population.
+ *
+ * Parsing is done via WHATWG `new URL(...)`. URLs that fail to parse (e.g.
+ * a malformed href scraped from the wild) still round-trip through
+ * `url_refs` — the natural key is the raw `url` string — but every
+ * decomposed column becomes `null` so filters that JOIN on `host` or
+ * `scheme` cleanly skip malformed rows instead of falsely matching.
+ *
+ * `port` is only populated when explicit in the URL. `new URL(...)`
+ * normalises the scheme's default port away (`https://example.com:443/` →
+ * `url.port === ''`), so we never synthesise a default. Two rows differing
+ * only by explicit vs implicit default port must not deduplicate anyway —
+ * `url_refs.url` (the raw string) is the natural key, not the decomposed
+ * columns.
+ *
+ * `query_hash` is the 32-byte content hash of the query string with the
+ * leading `?` stripped. Storing the raw query would defeat dedup on
+ * tracker URLs whose per-request keys explode dictionary size.
+ * @param url - Raw URL string (may be any WHATWG-parseable form, or malformed).
+ * @returns Every column that goes into `url_refs` alongside the raw URL.
+ */
+export declare function decomposeUrl(url: string): DecomposedUrl;

package/lib/archive/populate-ref-tables/decompose-url.js

@@ -0,0 +1,70 @@
+import { computeContentHash } from './compute-content-hash.js';
+/**
+ * URL schemes whose "pathname" is really an in-band opaque payload
+ * (base64-encoded image bytes, javascript source, blob URL fragment,
+ * etc.) rather than a routing key. For these schemes we leave
+ * `DecomposedUrl.path` as `null` so the indexed `url_refs.path` column
+ * does not balloon with per-URL opaque tails and defeat the dedup goal.
+ */
+const OPAQUE_PATH_SCHEMES = new Set([
+    'data',
+    'blob',
+    'javascript',
+    'about',
+]);
+/**
+ * Extracts `scheme` / `host` / `port` / `path` / `query_hash` / `fragment`
+ * columns from a URL string for `url_refs` population.
+ *
+ * Parsing is done via WHATWG `new URL(...)`. URLs that fail to parse (e.g.
+ * a malformed href scraped from the wild) still round-trip through
+ * `url_refs` — the natural key is the raw `url` string — but every
+ * decomposed column becomes `null` so filters that JOIN on `host` or
+ * `scheme` cleanly skip malformed rows instead of falsely matching.
+ *
+ * `port` is only populated when explicit in the URL. `new URL(...)`
+ * normalises the scheme's default port away (`https://example.com:443/` →
+ * `url.port === ''`), so we never synthesise a default. Two rows differing
+ * only by explicit vs implicit default port must not deduplicate anyway —
+ * `url_refs.url` (the raw string) is the natural key, not the decomposed
+ * columns.
+ *
+ * `query_hash` is the 32-byte content hash of the query string with the
+ * leading `?` stripped. Storing the raw query would defeat dedup on
+ * tracker URLs whose per-request keys explode dictionary size.
+ * @param url - Raw URL string (may be any WHATWG-parseable form, or malformed).
+ * @returns Every column that goes into `url_refs` alongside the raw URL.
+ */
+export function decomposeUrl(url) {
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        return {
+            scheme: null,
+            host: null,
+            port: null,
+            path: null,
+            query_hash: null,
+            fragment: null,
+        };
+    }
+    const scheme = parsed.protocol.slice(0, -1) || null;
+    const host = parsed.hostname === '' ? null : parsed.hostname.toLowerCase();
+    const port = parsed.port === '' ? null : Number.parseInt(parsed.port, 10);
+    const rawSearch = parsed.search.startsWith('?')
+        ? parsed.search.slice(1)
+        : parsed.search;
+    const query_hash = rawSearch === '' ? null : computeContentHash(rawSearch);
+    const fragment = parsed.hash === '' ? null : parsed.hash.slice(1);
+    const path = scheme !== null && OPAQUE_PATH_SCHEMES.has(scheme) ? null : parsed.pathname;
+    return {
+        scheme,
+        host,
+        port: port !== null && Number.isFinite(port) ? port : null,
+        path,
+        query_hash,
+        fragment,
+    };
+}

package/lib/archive/populate-ref-tables/header-stability.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Classifies a header name as volatile (excluded from `stable_hash`) or
+ * stable (included). Names are compared case-insensitively (lower-cased
+ * before lookup). Names not in {@link VOLATILE_HEADER_NAMES} default to
+ * **stable** — see the docs on that constant for why this direction is
+ * the safer default.
+ *
+ * An explicit stable list could be named instead (`content-type`,
+ * `content-length`, `cache-control`, `content-security-policy`,
+ * `x-frame-options`, `x-content-type-options`, `strict-transport-
+ * security`, `referrer-policy`, `permissions-policy`, `server`, `vary`,
+ * `location`, ...); we deliberately do NOT enumerate that set at runtime —
+ * every non-volatile header is stable by construction, and enumerating
+ * it would create a maintenance surface that the volatile-only lookup
+ * avoids.
+ * @param name - Header name (any case).
+ * @returns `true` when the header is volatile.
+ */
+export declare function isVolatileHeader(name: string): boolean;

package/lib/archive/populate-ref-tables/header-stability.js

@@ -0,0 +1,22 @@
+import { VOLATILE_HEADER_NAMES } from './volatile-header-names.js';
+/**
+ * Classifies a header name as volatile (excluded from `stable_hash`) or
+ * stable (included). Names are compared case-insensitively (lower-cased
+ * before lookup). Names not in {@link VOLATILE_HEADER_NAMES} default to
+ * **stable** — see the docs on that constant for why this direction is
+ * the safer default.
+ *
+ * An explicit stable list could be named instead (`content-type`,
+ * `content-length`, `cache-control`, `content-security-policy`,
+ * `x-frame-options`, `x-content-type-options`, `strict-transport-
+ * security`, `referrer-policy`, `permissions-policy`, `server`, `vary`,
+ * `location`, ...); we deliberately do NOT enumerate that set at runtime —
+ * every non-volatile header is stable by construction, and enumerating
+ * it would create a maintenance surface that the volatile-only lookup
+ * avoids.
+ * @param name - Header name (any case).
+ * @returns `true` when the header is volatile.
+ */
+export function isVolatileHeader(name) {
+    return VOLATILE_HEADER_NAMES.has(name.toLowerCase());
+}

package/lib/archive/populate-ref-tables/header-value-cache-key.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Composite cache key for `header_value_refs`: hex-encoded hash + a
+ * separator + raw value. Two values with the same hash but different
+ * strings (astronomically improbable, but a `BLOB` hash column is not
+ * a total function per SQL semantics) still resolve to distinct cache
+ * entries. Shared by the cache warmer
+ * ({@link ./create-header-table-caches.ts}) and the per-set upsert
+ * ({@link ./upsert-one-header-set.ts}) so both sides always agree on the
+ * key shape.
+ * @param hash - 32-byte content hash of `value`.
+ * @param value - Header value verbatim.
+ * @returns Cache key string.
+ * @example
+ * headerValueCacheKey(computeContentHash('no-cache'), 'no-cache');
+ * // '724cf9…|no-cache'
+ */
+export declare function headerValueCacheKey(hash: Buffer, value: string): string;

package/lib/archive/populate-ref-tables/header-value-cache-key.js

@@ -0,0 +1,19 @@
+/**
+ * Composite cache key for `header_value_refs`: hex-encoded hash + a
+ * separator + raw value. Two values with the same hash but different
+ * strings (astronomically improbable, but a `BLOB` hash column is not
+ * a total function per SQL semantics) still resolve to distinct cache
+ * entries. Shared by the cache warmer
+ * ({@link ./create-header-table-caches.ts}) and the per-set upsert
+ * ({@link ./upsert-one-header-set.ts}) so both sides always agree on the
+ * key shape.
+ * @param hash - 32-byte content hash of `value`.
+ * @param value - Header value verbatim.
+ * @returns Cache key string.
+ * @example
+ * headerValueCacheKey(computeContentHash('no-cache'), 'no-cache');
+ * // '724cf9…|no-cache'
+ */
+export function headerValueCacheKey(hash, value) {
+    return `${hash.toString('hex')}|${value}`;
+}

package/lib/archive/populate-ref-tables/normalize-mime.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Normalises a raw Content-Type header value to the canonical MIME form
+ * stored in `content_type_refs.normalized`: parameters (`; charset=...`)
+ * removed, C0 control characters (`0x00`..`0x1F`) and DEL (`0x7F`)
+ * stripped, trimmed, lower-cased.
+ *
+ * This is a derived column shared by two writers — the live-crawl upsert
+ * (`db-ops/_shared/upsert-content-type-ref.ts`) and the archive-migration
+ * populate (`populate-content-type-refs.ts`) — which MUST be
+ * same-input-same-output; a fork would split the dictionary so the same
+ * raw content-type maps to different normalized values depending on
+ * whether the row was crawled live or migrated. Keeping the single
+ * implementation here is what enforces that.
+ *
+ * Control chars are dropped character-by-character rather than via a
+ * regex literal so the source file carries no non-printing bytes and
+ * stays tool-friendly. `jsdom` / lax parsers occasionally emit them
+ * inside a raw Content-Type header (e.g. `text/html\r`).
+ * @param raw - Raw Content-Type header, guaranteed non-null non-empty.
+ * @returns Lower-cased MIME with parameters and control chars stripped.
+ * @example
+ * normalizeMime('Text/HTML; charset=UTF-8'); // => 'text/html'
+ */
+export declare function normalizeMime(raw: string): string;

package/lib/archive/populate-ref-tables/normalize-mime.js

@@ -0,0 +1,36 @@
+/**
+ * Normalises a raw Content-Type header value to the canonical MIME form
+ * stored in `content_type_refs.normalized`: parameters (`; charset=...`)
+ * removed, C0 control characters (`0x00`..`0x1F`) and DEL (`0x7F`)
+ * stripped, trimmed, lower-cased.
+ *
+ * This is a derived column shared by two writers — the live-crawl upsert
+ * (`db-ops/_shared/upsert-content-type-ref.ts`) and the archive-migration
+ * populate (`populate-content-type-refs.ts`) — which MUST be
+ * same-input-same-output; a fork would split the dictionary so the same
+ * raw content-type maps to different normalized values depending on
+ * whether the row was crawled live or migrated. Keeping the single
+ * implementation here is what enforces that.
+ *
+ * Control chars are dropped character-by-character rather than via a
+ * regex literal so the source file carries no non-printing bytes and
+ * stays tool-friendly. `jsdom` / lax parsers occasionally emit them
+ * inside a raw Content-Type header (e.g. `text/html\r`).
+ * @param raw - Raw Content-Type header, guaranteed non-null non-empty.
+ * @returns Lower-cased MIME with parameters and control chars stripped.
+ * @example
+ * normalizeMime('Text/HTML; charset=UTF-8'); // => 'text/html'
+ */
+export function normalizeMime(raw) {
+    const semi = raw.indexOf(';');
+    const head = semi === -1 ? raw : raw.slice(0, semi);
+    let output = '';
+    for (const ch of head) {
+        const code = ch.codePointAt(0);
+        if (code < 32 /* 0x20 = SPACE */ || code === 127 /* 0x7F = DEL */) {
+            continue;
+        }
+        output += ch;
+    }
+    return output.trim().toLowerCase();
+}

package/lib/archive/populate-ref-tables/populate-blob-refs.d.ts

@@ -0,0 +1,38 @@
+import type { ProgressCallback } from '../create-progress-reporter.js';
+import type { Knex } from 'knex';
+/**
+ * Populates `blob_refs` from every `images.src` / `images.currentSrc` /
+ * `resources.url` value that is a data URI longer than
+ * {@link DATA_URI_URL_REFS_LIMIT} bytes (issue #191).
+ *
+ * For each such value:
+ *
+ * 1. `decodeDataUri` strips the `data:...;base64,` (or `data:...,`) prefix
+ *    and returns the raw payload bytes.
+ * 2. `computeContentHash` hashes the payload bytes (32-byte SHA-256).
+ * 3. The payload is zstd-compressed (`codec='zstd'`) — matching
+ *    `page_html_blobs` / `json_refs`.
+ * 4. `INSERT OR IGNORE` on `hash` deduplicates: an `<img>` element and a
+ *    resource that share the same underlying base64 payload share one
+ *    `blob_refs` row.
+ *
+ * Malformed data URIs that `decodeDataUri` cannot decode are logged and
+ * skipped — the raw URI string still exists in the source row, and the
+ * migration script's operator can hunt it down from the warning log. The
+ * alternative — routing malformed large URIs back to `url_refs` — would
+ * break the "data URIs > threshold live in blob_refs" contract that 0.13
+ * lookups depend on.
+ *
+ * On the reference archive only ~429 images use data URIs, so the total
+ * dictionary is tiny — this step is I/O-cheap even without full
+ * chunking, but the same batching shape is used as elsewhere for
+ * consistency.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of each source table scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateBlobRefs(trx);
+ * });
+ */
+export declare function populateBlobRefs(trx: Knex, onProgress?: ProgressCallback): Promise<void>;

package/lib/archive/populate-ref-tables/populate-blob-refs.js

@@ -0,0 +1,134 @@
+import { zstdCompressSync } from 'node:zlib';
+import { createProgressReporter } from '../create-progress-reporter.js';
+import { computeContentHash } from './compute-content-hash.js';
+import { DATA_URI_URL_REFS_LIMIT } from './data-uri-url-refs-limit.js';
+import { decodeDataUri } from './decode-data-uri.js';
+/** Rows scanned per `SELECT`. See `populateTextRefs` for chunking rationale. */
+const READ_CHUNK_SIZE = 2000;
+/**
+ * Rows written per `INSERT`. blob_refs rows carry a compressed body BLOB
+ * so 100 rows keeps the total payload of a single INSERT statement well
+ * under WAL frame limits even for large SVGs.
+ */
+const INSERT_CHUNK_SIZE = 100;
+/**
+ * Source table + its URL-shaped column(s) that may hold a data URI large
+ * enough to route to `blob_refs`. `resources.url` is the resource's own
+ * identity (unlike `pages`/`content_items`, whose own URL is always a
+ * real http(s) address in practice) — a `<link>`/CSS sub-resource can
+ * legally be captured as an inline `data:` URI.
+ */
+const URL_SOURCES = [
+    { table: 'images', columns: ['src', 'currentSrc'] },
+    { table: 'resources', columns: ['url'] },
+];
+/**
+ * Populates `blob_refs` from every `images.src` / `images.currentSrc` /
+ * `resources.url` value that is a data URI longer than
+ * {@link DATA_URI_URL_REFS_LIMIT} bytes (issue #191).
+ *
+ * For each such value:
+ *
+ * 1. `decodeDataUri` strips the `data:...;base64,` (or `data:...,`) prefix
+ *    and returns the raw payload bytes.
+ * 2. `computeContentHash` hashes the payload bytes (32-byte SHA-256).
+ * 3. The payload is zstd-compressed (`codec='zstd'`) — matching
+ *    `page_html_blobs` / `json_refs`.
+ * 4. `INSERT OR IGNORE` on `hash` deduplicates: an `<img>` element and a
+ *    resource that share the same underlying base64 payload share one
+ *    `blob_refs` row.
+ *
+ * Malformed data URIs that `decodeDataUri` cannot decode are logged and
+ * skipped — the raw URI string still exists in the source row, and the
+ * migration script's operator can hunt it down from the warning log. The
+ * alternative — routing malformed large URIs back to `url_refs` — would
+ * break the "data URIs > threshold live in blob_refs" contract that 0.13
+ * lookups depend on.
+ *
+ * On the reference archive only ~429 images use data URIs, so the total
+ * dictionary is tiny — this step is I/O-cheap even without full
+ * chunking, but the same batching shape is used as elsewhere for
+ * consistency.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of each source table scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateBlobRefs(trx);
+ * });
+ */
+export async function populateBlobRefs(trx, onProgress) {
+    const seen = new Set();
+    const pending = [];
+    for (const source of URL_SOURCES) {
+        const hasTable = await trx.schema.hasTable(source.table);
+        if (!hasTable) {
+            continue;
+        }
+        const presentColumns = [];
+        for (const column of source.columns) {
+            if (await trx.schema.hasColumn(source.table, column)) {
+                presentColumns.push(column);
+            }
+        }
+        if (presentColumns.length === 0) {
+            continue;
+        }
+        const countRows = await trx(source.table).count({ n: '*' });
+        const total = Number(countRows[0]?.n ?? 0);
+        const report = createProgressReporter(`blob_refs (${source.table})`, total, onProgress);
+        let processed = 0;
+        let cursor = 0;
+        while (true) {
+            const rows = await trx(source.table)
+                .select('id', ...presentColumns)
+                .where('id', '>', cursor)
+                .orderBy('id', 'asc')
+                .limit(READ_CHUNK_SIZE);
+            if (rows.length === 0) {
+                break;
+            }
+            cursor = rows.at(-1).id;
+            processed += rows.length;
+            report(processed);
+            for (const row of rows) {
+                for (const column of presentColumns) {
+                    const value = row[column];
+                    if (value == null || value.length <= DATA_URI_URL_REFS_LIMIT) {
+                        continue;
+                    }
+                    if (!value.startsWith('data:')) {
+                        continue;
+                    }
+                    const decoded = decodeDataUri(value);
+                    if (decoded === null) {
+                        // eslint-disable-next-line no-console
+                        console.warn(`populateBlobRefs: skipping malformed data URI (length=${value.length}) — raw string still exists in ${source.table} row`);
+                        continue;
+                    }
+                    const hash = computeContentHash(decoded.bytes);
+                    const hex = hash.toString('hex');
+                    if (seen.has(hex)) {
+                        continue;
+                    }
+                    seen.add(hex);
+                    const compressed = zstdCompressSync(decoded.bytes);
+                    pending.push({
+                        hash,
+                        body: compressed,
+                        codec: 'zstd',
+                        size_raw: decoded.bytes.byteLength,
+                        size_stored: compressed.byteLength,
+                    });
+                    if (pending.length >= INSERT_CHUNK_SIZE) {
+                        await trx('blob_refs').insert(pending).onConflict('hash').ignore();
+                        pending.length = 0;
+                    }
+                }
+            }
+        }
+    }
+    if (pending.length > 0) {
+        await trx('blob_refs').insert(pending).onConflict('hash').ignore();
+    }
+}

package/lib/archive/populate-ref-tables/populate-content-type-refs.d.ts

@@ -0,0 +1,27 @@
+import type { Knex } from 'knex';
+/**
+ * Populates `content_type_refs` from every distinct `contentType` value
+ * currently stored in `pages` and `resources` (issue #191).
+ *
+ * Two independent DISTINCT SELECTs (one per table) are merged in JS
+ * rather than via SQL `UNION` — the cardinality is small in practice and
+ * doing it in JS avoids the knex-`union().select()` column-aliasing
+ * quirk (bare `.select()` on a union wraps in `SELECT *` and can lose
+ * the column name depending on driver version). Two per-table SELECTs
+ * are also fast because `contentType` is indexed on both tables via
+ * `idx_pages_listfilter` / natural column index (see `init-schema.ts`).
+ *
+ * `normalized` and `category` are derived in JS via {@link classifyContentType}
+ * so the rule table stays in one place; SQLite has no equivalent
+ * expression.
+ *
+ * `INSERT OR IGNORE` on the natural key `raw` makes this idempotent —
+ * re-running this populate after a partial failure never duplicates
+ * rows, only appends the new ones.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateContentTypeRefs(trx);
+ * });
+ */
+export declare function populateContentTypeRefs(trx: Knex): Promise<void>;

package/lib/archive/populate-ref-tables/populate-content-type-refs.js

@@ -0,0 +1,70 @@
+import { classifyContentType } from './classify-content-type.js';
+import { normalizeMime } from './normalize-mime.js';
+/**
+ * Distinct-`contentType` rows are collected from `pages` + `resources` in
+ * chunks of this size before being written into `content_type_refs`. The
+ * DISTINCT list is expected to be tiny (< 1000 rows even on the largest
+ * archives — the wire content-type space is small) so the chunk size only
+ * bounds worst-case parameter counts on `INSERT ... VALUES ...`.
+ */
+const INSERT_CHUNK_SIZE = 500;
+/**
+ * Populates `content_type_refs` from every distinct `contentType` value
+ * currently stored in `pages` and `resources` (issue #191).
+ *
+ * Two independent DISTINCT SELECTs (one per table) are merged in JS
+ * rather than via SQL `UNION` — the cardinality is small in practice and
+ * doing it in JS avoids the knex-`union().select()` column-aliasing
+ * quirk (bare `.select()` on a union wraps in `SELECT *` and can lose
+ * the column name depending on driver version). Two per-table SELECTs
+ * are also fast because `contentType` is indexed on both tables via
+ * `idx_pages_listfilter` / natural column index (see `init-schema.ts`).
+ *
+ * `normalized` and `category` are derived in JS via {@link classifyContentType}
+ * so the rule table stays in one place; SQLite has no equivalent
+ * expression.
+ *
+ * `INSERT OR IGNORE` on the natural key `raw` makes this idempotent —
+ * re-running this populate after a partial failure never duplicates
+ * rows, only appends the new ones.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateContentTypeRefs(trx);
+ * });
+ */
+export async function populateContentTypeRefs(trx) {
+    const distinctRaw = new Set();
+    if (await trx.schema.hasTable('pages')) {
+        const pagesRows = await trx('pages')
+            .distinct('contentType')
+            .whereNotNull('contentType');
+        for (const { contentType } of pagesRows) {
+            if (contentType != null && contentType !== '') {
+                distinctRaw.add(contentType);
+            }
+        }
+    }
+    if (await trx.schema.hasTable('resources')) {
+        const resourcesRows = await trx('resources')
+            .distinct('contentType')
+            .whereNotNull('contentType');
+        for (const { contentType } of resourcesRows) {
+            if (contentType != null && contentType !== '') {
+                distinctRaw.add(contentType);
+            }
+        }
+    }
+    if (distinctRaw.size === 0) {
+        return;
+    }
+    const inserts = [...distinctRaw].map((raw) => ({
+        raw,
+        normalized: normalizeMime(raw),
+        category: classifyContentType(raw),
+    }));
+    for (let index = 0; index < inserts.length; index += INSERT_CHUNK_SIZE) {
+        const chunk = inserts.slice(index, index + INSERT_CHUNK_SIZE);
+        await trx('content_type_refs').insert(chunk).onConflict('raw').ignore();
+    }
+}