@nitpicker/archive 0.0.0-stage → 0.25.0

package/lib/populate-ref-tables/decode-data-uri.js

@@ -0,0 +1,126 @@
+/**
+ * Attempts to decode a `data:...` URI into its raw payload bytes.
+ *
+ * Supports the two encoding forms in RFC 2397:
+ *
+ * - `data:<mime>;base64,<base64-bytes>` — the base64 tail is decoded via
+ *   `Buffer.from(..., 'base64')`.
+ * - `data:<mime>,<percent-encoded-bytes>` — the tail is percent-decoded
+ *   via `decodeURIComponent(...)`, then re-encoded as UTF-8. This form is
+ *   commonly used for SVG data URIs.
+ *
+ * Returns `null` on any parse / decode failure so the caller can skip the
+ * value without aborting the whole `blob_refs` population step. A single
+ * malformed data URI in a 470 K-row archive should not halt the migration
+ * — the raw `url` string is still preserved in the source `images` row.
+ * @param value - Raw column value; may or may not start with `data:`.
+ * @returns Decoded bytes, or `null` when the value isn't a data URI or
+ *   can't be decoded.
+ */
+export function decodeDataUri(value) {
+    if (!value.startsWith('data:')) {
+        return null;
+    }
+    const commaIndex = value.indexOf(',');
+    if (commaIndex === -1) {
+        return null;
+    }
+    const header = value.slice(5, commaIndex);
+    const payload = value.slice(commaIndex + 1);
+    try {
+        if (/;\s*base64$/i.test(header)) {
+            // `Buffer.from(..., 'base64')` silently strips characters outside
+            // the base64 alphabet — a malformed payload decodes to garbage
+            // instead of throwing. Validate the alphabet + padding shape
+            // first so callers can distinguish "successfully decoded" from
+            // "silently corrupted" and get the promised `null` on malformed
+            // input.
+            if (!isValidBase64(payload)) {
+                return null;
+            }
+            const bytes = Buffer.from(payload, 'base64');
+            return { bytes };
+        }
+        return { bytes: percentDecodeToBytes(payload) };
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Standard base64 alphabet + padding, plus one common URL-safe variant
+ * that occasionally appears in `data:` URIs even though RFC 2397 says
+ * only the standard alphabet. `-` / `_` are the URL-safe substitutes
+ * for `+` / `/`.
+ */
+const BASE64_ALPHABET_RE = /^[\w+/-]*={0,2}$/;
+/**
+ * Validates whether a base64 payload is well-formed enough that
+ * `Buffer.from(payload, 'base64')` won't need to silently strip
+ * characters. Whitespace-tolerant (RFC 2045 permits internal
+ * whitespace in base64 line-wrapped output).
+ * @param payload - Base64 payload (everything after `,` in the data URI).
+ * @returns `true` when the payload contains only valid base64 characters
+ *   + optional trailing padding.
+ */
+function isValidBase64(payload) {
+    const stripped = payload.replaceAll(/\s+/g, '');
+    if (stripped === '') {
+        return true;
+    }
+    if (!BASE64_ALPHABET_RE.test(stripped)) {
+        return false;
+    }
+    // base64 chunks group by 4 characters — a well-formed payload has
+    // length ≡ 0 (mod 4) once padding is applied. Padding-less trailing
+    // chunks of length 2 or 3 are also accepted (Node's decoder tolerates
+    // missing `=`), but length 1 or a padding-only chunk is malformed.
+    const remainder = stripped.length % 4;
+    if (remainder === 1) {
+        return false;
+    }
+    return true;
+}
+/**
+ * Percent-decodes a data-URI payload byte-by-byte, avoiding the round
+ * through `decodeURIComponent` → UTF-8 re-encode that mangles arbitrary
+ * binary payloads (e.g. a `%FF` byte becomes a JS string of code
+ * point U+00FF, then re-encodes to `[0xC3, 0xBF]` — the byte value
+ * changes). RFC 2397's non-base64 form treats the payload as octets,
+ * not as a UTF-8 string, so an octet-preserving decode is the correct
+ * inverse.
+ * @param payload - Raw payload text (everything after the `,`).
+ * @returns Payload bytes.
+ * @throws {URIError} If a `%` escape is truncated or not two hex
+ *   digits; the caller's try/catch maps this to a `null` return.
+ */
+function percentDecodeToBytes(payload) {
+    const chunks = [];
+    let i = 0;
+    while (i < payload.length) {
+        const ch = payload.codePointAt(i);
+        if (ch === 0x25 /* % */) {
+            if (i + 2 >= payload.length) {
+                throw new URIError('truncated % escape');
+            }
+            const hex = payload.slice(i + 1, i + 3);
+            if (!/^[\da-f]{2}$/i.test(hex)) {
+                throw new URIError('invalid % escape');
+            }
+            chunks.push(Buffer.from(hex, 'hex'));
+            i += 3;
+            continue;
+        }
+        if (ch > 127 /* 0x7F, ASCII high-bit boundary */) {
+            // A non-ASCII code point in a percent-encoded payload is
+            // unusual (browsers percent-encode outbound), but WHATWG
+            // tolerates it — encode as UTF-8 to preserve the character.
+            chunks.push(Buffer.from(payload[i], 'utf8'));
+            i += 1;
+            continue;
+        }
+        chunks.push(Buffer.from([ch]));
+        i += 1;
+    }
+    return Buffer.concat(chunks);
+}

package/lib/populate-ref-tables/decompose-header-set.d.ts

@@ -0,0 +1,29 @@
+import type { DecomposedHeaderSet } from './types.js';
+/**
+ * Parses one raw `responseHeaders` JSON string and produces the derived
+ * data shape required to insert into `header_sets`, `header_set_entries`,
+ * and `header_flags`.
+ *
+ * Behaviour:
+ *
+ * - `null` / `""` / `"null"` / `"{}"` / non-object JSON → returns `null`.
+ *   The caller sets `header_set_id = null` on the referring row.
+ * - Multiple values per header name (JSON arrays, e.g.
+ *   `{ "set-cookie": ["a=1", "b=2"] }`) become multiple
+ *   {@link HeaderEntry} rows with `occurrence` ordinals 1, 2, ... The
+ *   composite PK on `header_set_entries` `(header_set_id, name_id,
+ *   occurrence)` guarantees no truncation.
+ * - Names are lower-cased. Values are trimmed of whitespace only in the
+ *   sorted-hash construction; the raw value is preserved verbatim in
+ *   the emitted `HeaderEntry.value` (and hence in `header_value_refs`).
+ *   Trimming inside hash construction lets `content-type: text/html`
+ *   and `content-type:  text/html ` dedup to one stable set.
+ * - Sort order for hashing is `(name, occurrence)` in binary form —
+ *   deterministic regardless of the input JSON's insertion order or
+ *   how the JSON serializer arranged keys.
+ * @param rawJson - The raw JSON string exactly as stored in
+ *   `pages.responseHeaders` / `resources.responseHeaders`.
+ * @returns Decomposed shape, or `null` when the row has no meaningful
+ *   header set.
+ */
+export declare function decomposeHeaderSet(rawJson: string | null): DecomposedHeaderSet | null;

package/lib/populate-ref-tables/decompose-header-set.js

@@ -0,0 +1,157 @@
+import { computeContentHash } from './compute-content-hash.js';
+import { isVolatileHeader } from './header-stability.js';
+/**
+ * Parses one raw `responseHeaders` JSON string and produces the derived
+ * data shape required to insert into `header_sets`, `header_set_entries`,
+ * and `header_flags`.
+ *
+ * Behaviour:
+ *
+ * - `null` / `""` / `"null"` / `"{}"` / non-object JSON → returns `null`.
+ *   The caller sets `header_set_id = null` on the referring row.
+ * - Multiple values per header name (JSON arrays, e.g.
+ *   `{ "set-cookie": ["a=1", "b=2"] }`) become multiple
+ *   {@link HeaderEntry} rows with `occurrence` ordinals 1, 2, ... The
+ *   composite PK on `header_set_entries` `(header_set_id, name_id,
+ *   occurrence)` guarantees no truncation.
+ * - Names are lower-cased. Values are trimmed of whitespace only in the
+ *   sorted-hash construction; the raw value is preserved verbatim in
+ *   the emitted `HeaderEntry.value` (and hence in `header_value_refs`).
+ *   Trimming inside hash construction lets `content-type: text/html`
+ *   and `content-type:  text/html ` dedup to one stable set.
+ * - Sort order for hashing is `(name, occurrence)` in binary form —
+ *   deterministic regardless of the input JSON's insertion order or
+ *   how the JSON serializer arranged keys.
+ * @param rawJson - The raw JSON string exactly as stored in
+ *   `pages.responseHeaders` / `resources.responseHeaders`.
+ * @returns Decomposed shape, or `null` when the row has no meaningful
+ *   header set.
+ */
+export function decomposeHeaderSet(rawJson) {
+    if (rawJson == null || rawJson === '' || rawJson === 'null' || rawJson === '{}') {
+        return null;
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(rawJson);
+    }
+    catch {
+        return null;
+    }
+    if (parsed == null || typeof parsed !== 'object' || Array.isArray(parsed)) {
+        return null;
+    }
+    const entries = [];
+    // `occurrence` runs per (lower-cased name) across ALL sources for that
+    // name, so duplicate JSON keys that differ only in case
+    // (`{"Cookie":"a=1", "cookie":"b=2"}` — non-conforming but legal JSON,
+    // and the crawler stores whatever puppeteer hands it) don't collide on
+    // (name_id, occurrence). Without this counter the second entry would
+    // land at occurrence=1 too, violating the `header_set_entries` PK and
+    // silently getting dropped by INSERT OR IGNORE while `entry_count`
+    // still counted it.
+    const occurrenceByName = new Map();
+    for (const [rawName, rawValue] of Object.entries(parsed)) {
+        const name = rawName.toLowerCase();
+        const volatile = isVolatileHeader(name);
+        const nextOccurrence = () => {
+            const next = (occurrenceByName.get(name) ?? 0) + 1;
+            occurrenceByName.set(name, next);
+            return next;
+        };
+        if (Array.isArray(rawValue)) {
+            for (const one of rawValue) {
+                if (typeof one !== 'string') {
+                    continue;
+                }
+                entries.push({
+                    name,
+                    value: one,
+                    occurrence: nextOccurrence(),
+                    isVolatile: volatile,
+                });
+            }
+            continue;
+        }
+        if (typeof rawValue !== 'string') {
+            // parseResponseHeaders's stored shape is `Record<string, string |
+            // string[] | undefined>`, but hand-edited or non-conforming
+            // archives could smuggle in a nested object / number / boolean
+            // — coercing via `String()` would produce "[object Object]" and
+            // pollute `header_value_refs` with a garbage dictionary entry.
+            // Skipping is the conservative choice; the source row's raw
+            // JSON is still preserved via `raw_json_hash`.
+            continue;
+        }
+        entries.push({
+            name,
+            value: rawValue,
+            occurrence: nextOccurrence(),
+            isVolatile: volatile,
+        });
+    }
+    if (entries.length === 0) {
+        return null;
+    }
+    entries.sort((a, b) => {
+        if (a.name < b.name)
+            return -1;
+        if (a.name > b.name)
+            return 1;
+        return a.occurrence - b.occurrence;
+    });
+    // Canonicalize every entry exactly once, then partition the strings.
+    // Canonicalizing per hash pass instead (once in the all-entries pass,
+    // again in the stable / volatile passes) would serialize each entry
+    // twice, which becomes measurable on large archives where a single
+    // response can carry a couple of dozen entries.
+    const canonicalStrings = entries.map(canonicalize);
+    const stableCanonical = [];
+    const volatileCanonical = [];
+    let stableEntryCount = 0;
+    for (const [i, entry] of entries.entries()) {
+        if (entry.isVolatile) {
+            volatileCanonical.push(canonicalStrings[i]);
+        }
+        else {
+            stableCanonical.push(canonicalStrings[i]);
+            stableEntryCount += 1;
+        }
+    }
+    const rawJsonHash = computeContentHash(rawJson);
+    const rawHash = computeContentHash(canonicalStrings.join('\n'));
+    // A response with only volatile headers (tracker endpoints that return
+    // just `Date` / `Set-Cookie` / `Age`) hashes to `computeContentHash('')`
+    // — the canonical "empty stable profile" hash. All all-volatile
+    // responses cluster under this sentinel, which is the intended
+    // semantic (same empty stable profile = same cluster), and 0.13's
+    // DDL declares `stable_hash BLOB NOT NULL` so a `null` sentinel is not
+    // available even if we wanted one. Consumers that need to distinguish
+    // "empty stable set" from "populated stable set" should key off
+    // `stable_entry_count === 0` on the same row, not off the hash.
+    const stableHash = computeContentHash(stableCanonical.join('\n'));
+    const volatileHash = volatileCanonical.length === 0
+        ? null
+        : computeContentHash(volatileCanonical.join('\n'));
+    return {
+        rawJsonHash,
+        rawHash,
+        stableHash,
+        volatileHash,
+        entries,
+        entryCount: entries.length,
+        stableEntryCount,
+    };
+}
+/**
+ * Serializes one entry into the canonical `name/occurrence=value` form
+ * used inside the hash construction. The trailing occurrence keeps
+ * multi-value headers distinguishable while still letting single-value
+ * ones dedup naturally. Values are trimmed so surrounding whitespace
+ * does not fork the dedup key.
+ * @param entry - One decomposed header entry.
+ * @returns Canonical string joined into the hash pre-image.
+ */
+function canonicalize(entry) {
+    return `${entry.name}/${entry.occurrence}=${entry.value.trim()}`;
+}

package/lib/populate-ref-tables/decompose-url.d.ts

@@ -0,0 +1,25 @@
+import type { DecomposedUrl } from './types.js';
+/**
+ * Extracts `scheme` / `host` / `port` / `path` / `query_hash` / `fragment`
+ * columns from a URL string for `url_refs` population.
+ *
+ * Parsing is done via WHATWG `new URL(...)`. URLs that fail to parse (e.g.
+ * a malformed href scraped from the wild) still round-trip through
+ * `url_refs` — the natural key is the raw `url` string — but every
+ * decomposed column becomes `null` so filters that JOIN on `host` or
+ * `scheme` cleanly skip malformed rows instead of falsely matching.
+ *
+ * `port` is only populated when explicit in the URL. `new URL(...)`
+ * normalises the scheme's default port away (`https://example.com:443/` →
+ * `url.port === ''`), so we never synthesise a default. Two rows differing
+ * only by explicit vs implicit default port must not deduplicate anyway —
+ * `url_refs.url` (the raw string) is the natural key, not the decomposed
+ * columns.
+ *
+ * `query_hash` is the 32-byte content hash of the query string with the
+ * leading `?` stripped. Storing the raw query would defeat dedup on
+ * tracker URLs whose per-request keys explode dictionary size.
+ * @param url - Raw URL string (may be any WHATWG-parseable form, or malformed).
+ * @returns Every column that goes into `url_refs` alongside the raw URL.
+ */
+export declare function decomposeUrl(url: string): DecomposedUrl;

package/lib/populate-ref-tables/decompose-url.js

@@ -0,0 +1,70 @@
+import { computeContentHash } from './compute-content-hash.js';
+/**
+ * URL schemes whose "pathname" is really an in-band opaque payload
+ * (base64-encoded image bytes, javascript source, blob URL fragment,
+ * etc.) rather than a routing key. For these schemes we leave
+ * `DecomposedUrl.path` as `null` so the indexed `url_refs.path` column
+ * does not balloon with per-URL opaque tails and defeat the dedup goal.
+ */
+const OPAQUE_PATH_SCHEMES = new Set([
+    'data',
+    'blob',
+    'javascript',
+    'about',
+]);
+/**
+ * Extracts `scheme` / `host` / `port` / `path` / `query_hash` / `fragment`
+ * columns from a URL string for `url_refs` population.
+ *
+ * Parsing is done via WHATWG `new URL(...)`. URLs that fail to parse (e.g.
+ * a malformed href scraped from the wild) still round-trip through
+ * `url_refs` — the natural key is the raw `url` string — but every
+ * decomposed column becomes `null` so filters that JOIN on `host` or
+ * `scheme` cleanly skip malformed rows instead of falsely matching.
+ *
+ * `port` is only populated when explicit in the URL. `new URL(...)`
+ * normalises the scheme's default port away (`https://example.com:443/` →
+ * `url.port === ''`), so we never synthesise a default. Two rows differing
+ * only by explicit vs implicit default port must not deduplicate anyway —
+ * `url_refs.url` (the raw string) is the natural key, not the decomposed
+ * columns.
+ *
+ * `query_hash` is the 32-byte content hash of the query string with the
+ * leading `?` stripped. Storing the raw query would defeat dedup on
+ * tracker URLs whose per-request keys explode dictionary size.
+ * @param url - Raw URL string (may be any WHATWG-parseable form, or malformed).
+ * @returns Every column that goes into `url_refs` alongside the raw URL.
+ */
+export function decomposeUrl(url) {
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        return {
+            scheme: null,
+            host: null,
+            port: null,
+            path: null,
+            query_hash: null,
+            fragment: null,
+        };
+    }
+    const scheme = parsed.protocol.slice(0, -1) || null;
+    const host = parsed.hostname === '' ? null : parsed.hostname.toLowerCase();
+    const port = parsed.port === '' ? null : Number.parseInt(parsed.port, 10);
+    const rawSearch = parsed.search.startsWith('?')
+        ? parsed.search.slice(1)
+        : parsed.search;
+    const query_hash = rawSearch === '' ? null : computeContentHash(rawSearch);
+    const fragment = parsed.hash === '' ? null : parsed.hash.slice(1);
+    const path = scheme !== null && OPAQUE_PATH_SCHEMES.has(scheme) ? null : parsed.pathname;
+    return {
+        scheme,
+        host,
+        port: port !== null && Number.isFinite(port) ? port : null,
+        path,
+        query_hash,
+        fragment,
+    };
+}

package/lib/populate-ref-tables/header-stability.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Classifies a header name as volatile (excluded from `stable_hash`) or
+ * stable (included). Names are compared case-insensitively (lower-cased
+ * before lookup). Names not in {@link VOLATILE_HEADER_NAMES} default to
+ * **stable** — see the docs on that constant for why this direction is
+ * the safer default.
+ *
+ * An explicit stable list could be named instead (`content-type`,
+ * `content-length`, `cache-control`, `content-security-policy`,
+ * `x-frame-options`, `x-content-type-options`, `strict-transport-
+ * security`, `referrer-policy`, `permissions-policy`, `server`, `vary`,
+ * `location`, ...); we deliberately do NOT enumerate that set at runtime —
+ * every non-volatile header is stable by construction, and enumerating
+ * it would create a maintenance surface that the volatile-only lookup
+ * avoids.
+ * @param name - Header name (any case).
+ * @returns `true` when the header is volatile.
+ */
+export declare function isVolatileHeader(name: string): boolean;

package/lib/populate-ref-tables/header-stability.js

@@ -0,0 +1,22 @@
+import { VOLATILE_HEADER_NAMES } from './volatile-header-names.js';
+/**
+ * Classifies a header name as volatile (excluded from `stable_hash`) or
+ * stable (included). Names are compared case-insensitively (lower-cased
+ * before lookup). Names not in {@link VOLATILE_HEADER_NAMES} default to
+ * **stable** — see the docs on that constant for why this direction is
+ * the safer default.
+ *
+ * An explicit stable list could be named instead (`content-type`,
+ * `content-length`, `cache-control`, `content-security-policy`,
+ * `x-frame-options`, `x-content-type-options`, `strict-transport-
+ * security`, `referrer-policy`, `permissions-policy`, `server`, `vary`,
+ * `location`, ...); we deliberately do NOT enumerate that set at runtime —
+ * every non-volatile header is stable by construction, and enumerating
+ * it would create a maintenance surface that the volatile-only lookup
+ * avoids.
+ * @param name - Header name (any case).
+ * @returns `true` when the header is volatile.
+ */
+export function isVolatileHeader(name) {
+    return VOLATILE_HEADER_NAMES.has(name.toLowerCase());
+}

package/lib/populate-ref-tables/header-value-cache-key.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Composite cache key for `header_value_refs`: hex-encoded hash + a
+ * separator + raw value. Two values with the same hash but different
+ * strings (astronomically improbable, but a `BLOB` hash column is not
+ * a total function per SQL semantics) still resolve to distinct cache
+ * entries. Shared by the cache warmer
+ * ({@link ./create-header-table-caches.ts}) and the per-set upsert
+ * ({@link ./upsert-one-header-set.ts}) so both sides always agree on the
+ * key shape.
+ * @param hash - 32-byte content hash of `value`.
+ * @param value - Header value verbatim.
+ * @returns Cache key string.
+ * @example
+ * headerValueCacheKey(computeContentHash('no-cache'), 'no-cache');
+ * // '724cf9…|no-cache'
+ */
+export declare function headerValueCacheKey(hash: Buffer, value: string): string;

package/lib/populate-ref-tables/header-value-cache-key.js

@@ -0,0 +1,19 @@
+/**
+ * Composite cache key for `header_value_refs`: hex-encoded hash + a
+ * separator + raw value. Two values with the same hash but different
+ * strings (astronomically improbable, but a `BLOB` hash column is not
+ * a total function per SQL semantics) still resolve to distinct cache
+ * entries. Shared by the cache warmer
+ * ({@link ./create-header-table-caches.ts}) and the per-set upsert
+ * ({@link ./upsert-one-header-set.ts}) so both sides always agree on the
+ * key shape.
+ * @param hash - 32-byte content hash of `value`.
+ * @param value - Header value verbatim.
+ * @returns Cache key string.
+ * @example
+ * headerValueCacheKey(computeContentHash('no-cache'), 'no-cache');
+ * // '724cf9…|no-cache'
+ */
+export function headerValueCacheKey(hash, value) {
+    return `${hash.toString('hex')}|${value}`;
+}

package/lib/populate-ref-tables/normalize-mime.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Normalises a raw Content-Type header value to the canonical MIME form
+ * stored in `content_type_refs.normalized`: parameters (`; charset=...`)
+ * removed, C0 control characters (`0x00`..`0x1F`) and DEL (`0x7F`)
+ * stripped, trimmed, lower-cased.
+ *
+ * This is a derived column shared by two writers — the live-crawl upsert
+ * (`db-ops/_shared/upsert-content-type-ref.ts`) and the archive-migration
+ * populate (`populate-content-type-refs.ts`) — which MUST be
+ * same-input-same-output; a fork would split the dictionary so the same
+ * raw content-type maps to different normalized values depending on
+ * whether the row was crawled live or migrated. Keeping the single
+ * implementation here is what enforces that.
+ *
+ * Control chars are dropped character-by-character rather than via a
+ * regex literal so the source file carries no non-printing bytes and
+ * stays tool-friendly. `jsdom` / lax parsers occasionally emit them
+ * inside a raw Content-Type header (e.g. `text/html\r`).
+ * @param raw - Raw Content-Type header, guaranteed non-null non-empty.
+ * @returns Lower-cased MIME with parameters and control chars stripped.
+ * @example
+ * normalizeMime('Text/HTML; charset=UTF-8'); // => 'text/html'
+ */
+export declare function normalizeMime(raw: string): string;

package/lib/populate-ref-tables/normalize-mime.js

@@ -0,0 +1,36 @@
+/**
+ * Normalises a raw Content-Type header value to the canonical MIME form
+ * stored in `content_type_refs.normalized`: parameters (`; charset=...`)
+ * removed, C0 control characters (`0x00`..`0x1F`) and DEL (`0x7F`)
+ * stripped, trimmed, lower-cased.
+ *
+ * This is a derived column shared by two writers — the live-crawl upsert
+ * (`db-ops/_shared/upsert-content-type-ref.ts`) and the archive-migration
+ * populate (`populate-content-type-refs.ts`) — which MUST be
+ * same-input-same-output; a fork would split the dictionary so the same
+ * raw content-type maps to different normalized values depending on
+ * whether the row was crawled live or migrated. Keeping the single
+ * implementation here is what enforces that.
+ *
+ * Control chars are dropped character-by-character rather than via a
+ * regex literal so the source file carries no non-printing bytes and
+ * stays tool-friendly. `jsdom` / lax parsers occasionally emit them
+ * inside a raw Content-Type header (e.g. `text/html\r`).
+ * @param raw - Raw Content-Type header, guaranteed non-null non-empty.
+ * @returns Lower-cased MIME with parameters and control chars stripped.
+ * @example
+ * normalizeMime('Text/HTML; charset=UTF-8'); // => 'text/html'
+ */
+export function normalizeMime(raw) {
+    const semi = raw.indexOf(';');
+    const head = semi === -1 ? raw : raw.slice(0, semi);
+    let output = '';
+    for (const ch of head) {
+        const code = ch.codePointAt(0);
+        if (code < 32 /* 0x20 = SPACE */ || code === 127 /* 0x7F = DEL */) {
+            continue;
+        }
+        output += ch;
+    }
+    return output.trim().toLowerCase();
+}

package/lib/populate-ref-tables/populate-blob-refs.d.ts

@@ -0,0 +1,38 @@
+import type { ProgressCallback } from '../create-progress-reporter.js';
+import type { Knex } from 'knex';
+/**
+ * Populates `blob_refs` from every `images.src` / `images.currentSrc` /
+ * `resources.url` value that is a data URI longer than
+ * {@link DATA_URI_URL_REFS_LIMIT} bytes (issue #191).
+ *
+ * For each such value:
+ *
+ * 1. `decodeDataUri` strips the `data:...;base64,` (or `data:...,`) prefix
+ *    and returns the raw payload bytes.
+ * 2. `computeContentHash` hashes the payload bytes (32-byte SHA-256).
+ * 3. The payload is zstd-compressed (`codec='zstd'`) — matching
+ *    `page_html_blobs` / `json_refs`.
+ * 4. `INSERT OR IGNORE` on `hash` deduplicates: an `<img>` element and a
+ *    resource that share the same underlying base64 payload share one
+ *    `blob_refs` row.
+ *
+ * Malformed data URIs that `decodeDataUri` cannot decode are logged and
+ * skipped — the raw URI string still exists in the source row, and the
+ * migration script's operator can hunt it down from the warning log. The
+ * alternative — routing malformed large URIs back to `url_refs` — would
+ * break the "data URIs > threshold live in blob_refs" contract that 0.13
+ * lookups depend on.
+ *
+ * On the reference archive only ~429 images use data URIs, so the total
+ * dictionary is tiny — this step is I/O-cheap even without full
+ * chunking, but the same batching shape is used as elsewhere for
+ * consistency.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of each source table scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateBlobRefs(trx);
+ * });
+ */
+export declare function populateBlobRefs(trx: Knex, onProgress?: ProgressCallback): Promise<void>;