@nitpicker/crawler 0.11.0 → 0.13.0

package/lib/archive/populate-ref-tables/populate-url-refs.js

@@ -0,0 +1,148 @@
+import { createProgressReporter } from '../create-progress-reporter.js';
+import { DATA_URI_URL_REFS_LIMIT } from './data-uri-url-refs-limit.js';
+import { decomposeUrl } from './decompose-url.js';
+/**
+ * Rows inserted per `INSERT INTO url_refs ... VALUES (...)` statement. Kept
+ * well under SQLite's `SQLITE_MAX_VARIABLE_NUMBER` (default 32766 on
+ * modern builds) — each row binds 7 parameters (url + scheme + host +
+ * port + path + query_hash + fragment), so 500 rows = 3500 params.
+ */
+const INSERT_CHUNK_SIZE = 500;
+/**
+ * Rows scanned per keyset-paginated `SELECT` chunk. One SELECT reads all
+ * URL columns for the source table's chunk simultaneously so the pages
+ * table is scanned exactly once per populate call, not N-times-once-
+ * per-URL-column — a per-column scan would multiply migration wall-clock
+ * by the count of URL-shaped columns.
+ */
+const READ_CHUNK_SIZE = 5000;
+/**
+ * Source table + its URL-shaped columns. Each column is scanned in
+ * lock-step per row, then filtered through
+ * {@link routeDataUriAwayFromUrlRefs} so large data URIs skip `url_refs`
+ * and route to `blob_refs` instead (handled by
+ * {@link ./populate-blob-refs.ts}).
+ *
+ * The filter is applied to EVERY URL column (not just image columns):
+ * `og:image`, `icon_href`, `apple-touch-icon`, `twitter:image`, and even
+ * `og:url` / `canonical` can legally hold data URIs in the wild, and a
+ * >512-byte data URI in any of those must land in `blob_refs`, not
+ * pollute the URL dictionary.
+ */
+const URL_SOURCES = [
+    {
+        table: 'pages',
+        columns: [
+            'url',
+            'canonical',
+            'og_url',
+            'og_image',
+            'icon_href',
+            'appleTouchIcon_href',
+            'amphtml',
+            'manifest',
+            'twitter_image',
+        ],
+    },
+    { table: 'resources', columns: ['url'] },
+    { table: 'images', columns: ['src', 'currentSrc'] },
+];
+/**
+ * Predicate applied to every URL column value: keep the value in the
+ * `url_refs` stream unless it is a data URI larger than
+ * {@link DATA_URI_URL_REFS_LIMIT}. Large data URIs route to `blob_refs`
+ * instead (via {@link ./populate-blob-refs.ts}).
+ * @param value - Raw column value (non-null, non-empty by contract).
+ * @returns `true` if the value belongs in `url_refs`.
+ */
+function routeDataUriAwayFromUrlRefs(value) {
+    if (!value.startsWith('data:')) {
+        return true;
+    }
+    return value.length <= DATA_URI_URL_REFS_LIMIT;
+}
+/**
+ * Populates `url_refs` from every URL-shaped column across `pages`,
+ * `resources`, and `images` (issue #191).
+ *
+ * Runs two passes per source table:
+ *
+ * 1. **Collect distinct URLs** into an in-process `Set<string>`, streaming
+ *    rows from each source table by keyset pagination on `id`. All the
+ *    table's URL columns are read in a single SELECT so the pages scan
+ *    is O(rows), not O(rows × columns). Peak memory is bounded by the
+ *    count of distinct URLs, not the row count.
+ * 2. **Bulk-insert** into `url_refs` in chunks of {@link INSERT_CHUNK_SIZE},
+ *    with the decomposed columns (`scheme` / `host` / `port` / `path` /
+ *    `query_hash` / `fragment`) derived in JS by {@link decomposeUrl}.
+ *
+ * `INSERT OR IGNORE` on `url_refs.url` makes the step idempotent —
+ * repeated invocations after partial failure only add new URLs.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of each source table scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateUrlRefs(trx);
+ * });
+ */
+export async function populateUrlRefs(trx, onProgress) {
+    const seen = new Set();
+    for (const source of URL_SOURCES) {
+        const hasTable = await trx.schema.hasTable(source.table);
+        if (!hasTable) {
+            continue;
+        }
+        const presentColumns = [];
+        for (const column of source.columns) {
+            if (await trx.schema.hasColumn(source.table, column)) {
+                presentColumns.push(column);
+            }
+        }
+        if (presentColumns.length === 0) {
+            continue;
+        }
+        const countRows = await trx(source.table).count({ n: '*' });
+        const total = Number(countRows[0]?.n ?? 0);
+        const report = createProgressReporter(`url_refs (${source.table})`, total, onProgress);
+        let processed = 0;
+        let cursor = 0;
+        while (true) {
+            const rows = await trx(source.table)
+                .select('id', ...presentColumns)
+                .where('id', '>', cursor)
+                .orderBy('id', 'asc')
+                .limit(READ_CHUNK_SIZE);
+            if (rows.length === 0) {
+                break;
+            }
+            cursor = rows.at(-1).id;
+            processed += rows.length;
+            report(processed);
+            for (const row of rows) {
+                for (const column of presentColumns) {
+                    const value = row[column];
+                    if (typeof value !== 'string' || value === '') {
+                        continue;
+                    }
+                    if (!routeDataUriAwayFromUrlRefs(value)) {
+                        continue;
+                    }
+                    seen.add(value);
+                }
+            }
+        }
+    }
+    if (seen.size === 0) {
+        return;
+    }
+    const inserts = [];
+    for (const url of seen) {
+        const decomposed = decomposeUrl(url);
+        inserts.push({ url, ...decomposed });
+    }
+    for (let index = 0; index < inserts.length; index += INSERT_CHUNK_SIZE) {
+        const chunk = inserts.slice(index, index + INSERT_CHUNK_SIZE);
+        await trx('url_refs').insert(chunk).onConflict('url').ignore();
+    }
+}

package/lib/archive/populate-ref-tables/test-utils/count-rows.d.ts

@@ -0,0 +1,15 @@
+import type { Knex } from 'knex';
+/**
+ * Runs `SELECT count(?) FROM <table>` and returns the count as a plain
+ * JS number. Exists purely to keep spec files inside populate-ref-tables/ compliant
+ * with `unicorn/no-await-expression-member` — inlining
+ * `Number((await db(t).count(...))[0]!.n)` reads fine but violates the
+ * rule at every callsite.
+ * @param db - Knex instance (typically a spec-local in-memory DB).
+ * @param table - Table name.
+ * @param countColumn - Column to count; defaults to `'id'` because every
+ *   populate-refs table except `header_set_entries` / `header_flags` uses
+ *   that PK name. Pass `'header_set_id'` for the two exceptions.
+ * @returns Row count.
+ */
+export declare function countRows(db: Knex, table: string, countColumn?: string): Promise<number>;

package/lib/archive/populate-ref-tables/test-utils/count-rows.js

@@ -0,0 +1,17 @@
+/**
+ * Runs `SELECT count(?) FROM <table>` and returns the count as a plain
+ * JS number. Exists purely to keep spec files inside populate-ref-tables/ compliant
+ * with `unicorn/no-await-expression-member` — inlining
+ * `Number((await db(t).count(...))[0]!.n)` reads fine but violates the
+ * rule at every callsite.
+ * @param db - Knex instance (typically a spec-local in-memory DB).
+ * @param table - Table name.
+ * @param countColumn - Column to count; defaults to `'id'` because every
+ *   populate-refs table except `header_set_entries` / `header_flags` uses
+ *   that PK name. Pass `'header_set_id'` for the two exceptions.
+ * @returns Row count.
+ */
+export async function countRows(db, table, countColumn = 'id') {
+    const rows = await db(table).count({ n: countColumn });
+    return Number(rows[0].n);
+}

package/lib/archive/populate-ref-tables/types.d.ts

@@ -0,0 +1,197 @@
+/**
+ * Domain types for the 0.13 population step (issue #191). Every
+ * shared interface / type alias lives here per the repo-wide
+ * `型は types.ts に集約` rule; implementation modules export functions
+ * only.
+ */
+/**
+ * Decomposed URL components that populate `url_refs.{scheme,host,port,path,
+ * query_hash,fragment}`. All slots are populated in a single pass over the
+ * `url` string so callers do not have to re-parse per column.
+ */
+export interface DecomposedUrl {
+    /**
+     * URL scheme without the trailing colon (`https`, `http`, `mailto`, ...).
+     * `null` for URLs that fail `new URL(...)` parsing.
+     */
+    scheme: string | null;
+    /**
+     * Host lower-cased (`www.example.com`). `null` when the URL has no
+     * `authority` component (e.g. `mailto:`, `data:`, `javascript:`), and
+     * `null` on parse failure.
+     */
+    host: string | null;
+    /**
+     * TCP port when explicit in the URL; `null` when the URL uses the
+     * scheme default (WHATWG URL normalises default ports away — we do NOT
+     * synthesise a default here because two rows differing only by explicit
+     * vs implicit `:443` should hash to the same URL, and `url_refs.url` is
+     * the natural key, not this column).
+     */
+    port: number | null;
+    /**
+     * Path portion (`/foo/bar`). `null` when the URL has no path or the
+     * URL uses an opaque scheme whose "pathname" is really an in-band
+     * payload (`data:`, `blob:`, `javascript:`) — see
+     * {@link ../decompose-url.ts} for the opaque-scheme handling.
+     */
+    path: string | null;
+    /**
+     * 32-byte content hash of the raw query string exactly as it appears in
+     * the URL (`?a=1&b=2` — the leading `?` is stripped before hashing).
+     * `null` when the URL has no query string, when the query string is
+     * empty, or on parse failure.
+     *
+     * Storing the raw query would defeat dedup on tracker URLs whose
+     * per-request query keys blow up dictionary size; the hash retains the
+     * same-URL-⇒-same-row property without the payload.
+     */
+    query_hash: Buffer | null;
+    /**
+     * Fragment (`#section-2`) with the leading `#` stripped. `null` when
+     * the URL has no fragment or on parse failure.
+     */
+    fragment: string | null;
+}
+/**
+ * Successful decode of a `data:` URI: the raw payload bytes.
+ */
+export interface DecodedDataUri {
+    /** Payload bytes after base64 or percent-decoding. */
+    bytes: Buffer;
+}
+/**
+ * One header entry after decomposition — flattened out of the parsed JSON
+ * `Record<name, string | string[]>` so that:
+ *
+ * - Multiple values for the same name (e.g. two `Set-Cookie` lines) each
+ *   get their own entry with a distinct `occurrence` ordinal starting at 1.
+ * - `isVolatile` is pre-computed once per entry (rather than re-looked-up
+ *   at hash time) so both `stableHash` and the `header_set_entries`
+ *   inserts see identical classification.
+ */
+export interface HeaderEntry {
+    /** Lower-cased header name; comparison and dedup keys always use this form. */
+    name: string;
+    /** Raw header value (each element of `string[]` inputs becomes one entry). */
+    value: string;
+    /**
+     * 1-based ordinal among entries sharing the same `name` within one
+     * response. Always `1` for single-value headers; ≥ 2 when a name
+     * repeats (e.g. multiple `Set-Cookie` lines).
+     */
+    occurrence: number;
+    /** Pre-computed volatility. */
+    isVolatile: boolean;
+}
+/**
+ * Result of decomposing one raw `responseHeaders` JSON string into the
+ * shape required for `header_sets`, `header_set_entries`, and
+ * `header_flags` inserts. `null` results from the decompose helper
+ * indicate "no header set to insert" (null / `{}` / parse failure);
+ * callers write `header_set_id = null` to the referring row.
+ */
+export interface DecomposedHeaderSet {
+    /**
+     * 32-byte hash of the raw JSON string exactly as stored on the source
+     * row. Populates `header_sets.raw_json_hash` — a temporary column used
+     * by the 0.13 lookup to map an old `pages.responseHeaders` /
+     * `resources.responseHeaders` value back to its `header_sets.id`
+     * without any SQL function call.
+     */
+    rawJsonHash: Buffer;
+    /**
+     * 32-byte hash of the sorted `name=value` pairs of **all** entries
+     * (stable + volatile). Populates `header_sets.raw_hash` — a UNIQUE
+     * column so a JS-side upsert can dedup on identical decoded sets.
+     */
+    rawHash: Buffer;
+    /**
+     * 32-byte hash of the sorted `name=value` pairs of stable entries
+     * only. Populates `header_sets.stable_hash` — indexed so 0.13
+     * readers can answer "how many responses share this stable header
+     * profile" without scanning every entry.
+     */
+    stableHash: Buffer;
+    /**
+     * 32-byte hash of the sorted `name=value` pairs of volatile entries
+     * only, or `null` when the set has no volatile entries. Populates
+     * `header_sets.volatile_hash`.
+     */
+    volatileHash: Buffer | null;
+    /** Every decomposed entry, ordered by (name, occurrence). */
+    entries: readonly HeaderEntry[];
+    /** Total entry count — matches `header_sets.entry_count`. */
+    entryCount: number;
+    /**
+     * Number of stable entries — matches `header_sets.stable_entry_count`.
+     * `volatile_entry_count` is derived as `entry_count - stable_entry_count`
+     * and not stored separately.
+     */
+    stableEntryCount: number;
+}
+/**
+ * Shape written into `header_flags` for one `header_set_id`. `has_*`
+ * columns are `INTEGER NOT NULL` (0/1) per the 0.13 DDL;
+ * `cache_policy` is nullable and holds a compact summary of the
+ * `Cache-Control` value (or `null` when absent).
+ */
+export interface HeaderFlagsRow {
+    /** `1` when at least one entry's name equals `content-security-policy`. */
+    has_csp: 0 | 1;
+    /** `1` when at least one entry's name equals `x-frame-options`. */
+    has_x_frame_options: 0 | 1;
+    /** `1` when at least one entry's name equals `x-content-type-options`. */
+    has_x_content_type_options: 0 | 1;
+    /** `1` when at least one entry's name equals `strict-transport-security`. */
+    has_hsts: 0 | 1;
+    /** `1` when at least one entry's name equals `referrer-policy`. */
+    has_referrer_policy: 0 | 1;
+    /** `1` when at least one entry's name equals `permissions-policy`. */
+    has_permissions_policy: 0 | 1;
+    /** `1` when at least one entry's name equals `set-cookie`. */
+    has_set_cookie: 0 | 1;
+    /**
+     * The `cache-control` value, verbatim, when present; `null` when
+     * absent. Multi-value `Cache-Control` headers are joined with `, ` so
+     * the summary captures every directive without losing per-occurrence
+     * context.
+     */
+    cache_policy: string | null;
+}
+/**
+ * In-process caches shared by every header-table upsert against one
+ * archive connection. The three id-maps resolve `header_name_refs.id` /
+ * `header_value_refs.id` / `header_sets.id` without hitting the DB once
+ * a value has been seen (or preloaded from existing rows).
+ *
+ * `setIdsProcessedThisRun` tracks the subset of `setIdByRawJsonHash` /
+ * `setIdByRawHash` that the current process has already written
+ * entries + flags for. A preloaded id from a prior run is present in the
+ * id caches but NOT in this set — so the entries + flags inserts still
+ * run once (safe under `INSERT OR IGNORE`) to repair any missing rows
+ * left by a crashed writer. Once entries + flags for a given setId have
+ * been written this run, the shortcut skips redundant no-op inserts.
+ */
+export interface HeaderTableCaches {
+    /** `lower-cased name → header_name_refs.id`. */
+    readonly nameIdCache: Map<string, number>;
+    /** `hexHash|value → header_value_refs.id`. */
+    readonly valueIdCache: Map<string, number>;
+    /** `hex(raw_json_hash) → header_sets.id`. */
+    readonly setIdByRawJsonHash: Map<string, number>;
+    /** `hex(raw_hash) → header_sets.id`. */
+    readonly setIdByRawHash: Map<string, number>;
+    /** `header_sets.id` values already fully processed by this process. */
+    readonly setIdsProcessedThisRun: Set<number>;
+}
+/**
+ * Category label attached to each content-type rule. Mirrors the union
+ * `ContentTypeCategory` in `@nitpicker/query/src/types.ts` — kept as a
+ * plain string literal here so `@nitpicker/crawler`'s 0.13 step
+ * does not need to depend on `@nitpicker/query` (which itself depends
+ * on this package). If the two copies drift, `classify-content-type.spec.ts`
+ * here and `content-type-rules.spec.ts` on the query side will disagree
+ * on the same fixture.
+ */
+export type ContentTypeCategory = 'html' | 'pdf' | 'csv' | 'word' | 'excel' | 'powerpoint' | 'image' | 'audio' | 'video' | 'font' | 'css' | 'javascript' | 'json' | 'xml' | 'archive' | 'text' | 'other' | 'unknown';

package/lib/archive/populate-ref-tables/types.js

@@ -0,0 +1,7 @@
+/**
+ * Domain types for the 0.13 population step (issue #191). Every
+ * shared interface / type alias lives here per the repo-wide
+ * `型は types.ts に集約` rule; implementation modules export functions
+ * only.
+ */
+export {};

package/lib/archive/populate-ref-tables/upsert-one-header-set.d.ts

@@ -0,0 +1,34 @@
+import type { DecomposedHeaderSet, HeaderTableCaches } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Upserts one decomposed header set and its entries + flags, returning
+ * the `header_sets.id`. Shared by the bulk archive populate
+ * ({@link ./populate-header-tables.ts}) and the crawler's per-response
+ * write path, so both produce byte-identical header rows for the same
+ * raw `responseHeaders` value.
+ *
+ * Set-id resolution order:
+ *
+ * 1. `raw_json_hash` cache — same JSON key ordering as a previous row.
+ * 2. `raw_hash` cache — same sorted-entries identity, different JSON key
+ *    ordering. This branch links the new `raw_json_hash` to the existing
+ *    setId in the in-process map so subsequent same-JSON hits fast-path.
+ * 3. Otherwise INSERT a new `header_sets` row.
+ *
+ * Entries + flags are inserted once per `setId` per process (guarded by
+ * `setIdsProcessedThisRun`); already-inserted rows short-circuit
+ * cheaply, and preloaded-but-unprocessed setIds (partial prior run) get
+ * their entries / flags repaired via `INSERT OR IGNORE`.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param decomposed - Result of `decomposeHeaderSet` (never `null` — the
+ *   caller short-circuits null decompositions to `header_set_id = null`).
+ * @param caches - The in-process caches, mutated in place. Create via
+ *   {@link ./create-header-table-caches.ts}.
+ * @returns The `header_sets.id` for this set.
+ * @example
+ * const decomposed = decomposeHeaderSet(rawJson);
+ * if (decomposed !== null) {
+ *   const setId = await upsertOneHeaderSet(knex, decomposed, caches);
+ * }
+ */
+export declare function upsertOneHeaderSet(trx: Knex, decomposed: DecomposedHeaderSet, caches: HeaderTableCaches): Promise<number>;

package/lib/archive/populate-ref-tables/upsert-one-header-set.js

@@ -0,0 +1,208 @@
+import { computeContentHash } from './compute-content-hash.js';
+import { computeHeaderFlags } from './compute-header-flags.js';
+import { headerValueCacheKey } from './header-value-cache-key.js';
+/**
+ * Rows inserted per INSERT statement into `header_set_entries`. Each row
+ * binds 5 params so 500 rows = 2500 params — safely under SQLite's
+ * default variable limit.
+ */
+const ENTRY_INSERT_CHUNK_SIZE = 500;
+/**
+ * Upserts one decomposed header set and its entries + flags, returning
+ * the `header_sets.id`. Shared by the bulk archive populate
+ * ({@link ./populate-header-tables.ts}) and the crawler's per-response
+ * write path, so both produce byte-identical header rows for the same
+ * raw `responseHeaders` value.
+ *
+ * Set-id resolution order:
+ *
+ * 1. `raw_json_hash` cache — same JSON key ordering as a previous row.
+ * 2. `raw_hash` cache — same sorted-entries identity, different JSON key
+ *    ordering. This branch links the new `raw_json_hash` to the existing
+ *    setId in the in-process map so subsequent same-JSON hits fast-path.
+ * 3. Otherwise INSERT a new `header_sets` row.
+ *
+ * Entries + flags are inserted once per `setId` per process (guarded by
+ * `setIdsProcessedThisRun`); already-inserted rows short-circuit
+ * cheaply, and preloaded-but-unprocessed setIds (partial prior run) get
+ * their entries / flags repaired via `INSERT OR IGNORE`.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param decomposed - Result of `decomposeHeaderSet` (never `null` — the
+ *   caller short-circuits null decompositions to `header_set_id = null`).
+ * @param caches - The in-process caches, mutated in place. Create via
+ *   {@link ./create-header-table-caches.ts}.
+ * @returns The `header_sets.id` for this set.
+ * @example
+ * const decomposed = decomposeHeaderSet(rawJson);
+ * if (decomposed !== null) {
+ *   const setId = await upsertOneHeaderSet(knex, decomposed, caches);
+ * }
+ */
+export async function upsertOneHeaderSet(trx, decomposed, caches) {
+    const rawJsonHashHex = decomposed.rawJsonHash.toString('hex');
+    const rawHashHex = decomposed.rawHash.toString('hex');
+    let setId = caches.setIdByRawJsonHash.get(rawJsonHashHex);
+    if (setId === undefined) {
+        setId = caches.setIdByRawHash.get(rawHashHex);
+        if (setId === undefined) {
+            setId = await insertHeaderSet(trx, decomposed);
+            caches.setIdByRawJsonHash.set(rawJsonHashHex, setId);
+            caches.setIdByRawHash.set(rawHashHex, setId);
+        }
+        else {
+            // Same sorted-entries identity as an existing row, different JSON
+            // key ordering. Reuse the existing setId; the raw_json_hash of
+            // this variant has no persistent home in the 0.13 schema
+            // (raw_json_hash is UNIQUE per row) — linking it in the
+            // in-process map is enough for id resolution.
+            caches.setIdByRawJsonHash.set(rawJsonHashHex, setId);
+        }
+    }
+    if (caches.setIdsProcessedThisRun.has(setId)) {
+        return setId;
+    }
+    const nameIds = await resolveNameIds(trx, decomposed.entries, caches.nameIdCache);
+    const valueIds = await resolveValueIds(trx, decomposed.entries, caches.valueIdCache);
+    const entryRows = decomposed.entries.map((entry, index) => ({
+        header_set_id: setId,
+        name_id: nameIds[index],
+        occurrence: entry.occurrence,
+        value_id: valueIds[index],
+        is_volatile: entry.isVolatile ? 1 : 0,
+    }));
+    for (let index = 0; index < entryRows.length; index += ENTRY_INSERT_CHUNK_SIZE) {
+        const chunk = entryRows.slice(index, index + ENTRY_INSERT_CHUNK_SIZE);
+        await trx('header_set_entries')
+            .insert(chunk)
+            .onConflict(['header_set_id', 'name_id', 'occurrence'])
+            .ignore();
+    }
+    const flags = computeHeaderFlags(decomposed.entries);
+    await trx('header_flags')
+        .insert({ header_set_id: setId, ...flags })
+        .onConflict('header_set_id')
+        .ignore();
+    caches.setIdsProcessedThisRun.add(setId);
+    return setId;
+}
+/**
+ * Inserts a new `header_sets` row and returns its id. Neither
+ * `raw_json_hash` nor `raw_hash` collided with an existing row when
+ * this is called — the caller has already checked both caches.
+ * @param trx - Knex instance.
+ * @param decomposed - Result of `decomposeHeaderSet`.
+ * @returns The newly-inserted `header_sets.id`.
+ */
+async function insertHeaderSet(trx, decomposed) {
+    const rows = await trx('header_sets')
+        .insert({
+        raw_json_hash: decomposed.rawJsonHash,
+        raw_hash: decomposed.rawHash,
+        stable_hash: decomposed.stableHash,
+        volatile_hash: decomposed.volatileHash,
+        entry_count: decomposed.entryCount,
+        stable_entry_count: decomposed.stableEntryCount,
+    })
+        .returning('id');
+    const first = rows[0];
+    if (first === undefined) {
+        throw new Error('upsertOneHeaderSet: header_sets INSERT returned no rows despite absent conflict');
+    }
+    return first.id;
+}
+/**
+ * Resolves the `name_id` for every entry in `entries`, inserting new
+ * names into `header_name_refs` as needed. Mutates `cache` with any new
+ * ids.
+ * @param trx - Knex instance.
+ * @param entries - Decomposed entries.
+ * @param cache - Name → id cache; mutated in place.
+ * @returns Parallel array of name ids (same order as `entries`).
+ */
+async function resolveNameIds(trx, entries, cache) {
+    const missing = new Set();
+    for (const entry of entries) {
+        if (!cache.has(entry.name)) {
+            missing.add(entry.name);
+        }
+    }
+    if (missing.size > 0) {
+        const missingNames = [...missing];
+        // Chunked like the entries insert below: knex compiles a multi-row
+        // insert-with-onConflict into a compound SELECT, and SQLite rejects
+        // more than 500 compound terms per statement.
+        for (let index = 0; index < missingNames.length; index += ENTRY_INSERT_CHUNK_SIZE) {
+            const chunk = missingNames.slice(index, index + ENTRY_INSERT_CHUNK_SIZE);
+            await trx('header_name_refs')
+                .insert(chunk.map((name) => ({ name })))
+                .onConflict('name')
+                .ignore();
+            const inserted = await trx('header_name_refs')
+                .select('id', 'name')
+                .whereIn('name', chunk);
+            for (const row of inserted) {
+                cache.set(row.name, row.id);
+            }
+        }
+    }
+    return entries.map((entry) => {
+        const id = cache.get(entry.name);
+        if (id === undefined) {
+            throw new Error(`upsertOneHeaderSet: name_id not resolved for ${entry.name}`);
+        }
+        return id;
+    });
+}
+/**
+ * Resolves the `value_id` for every entry in `entries`, inserting new
+ * values into `header_value_refs` as needed. Values are hashed once and
+ * deduped by `(hash, value)` per the 0.13 UNIQUE constraint.
+ *
+ * The read-back after INSERT narrows on `.whereIn('value', ...)` — the
+ * string values just written — instead of binding the Buffer hashes into
+ * a `WHERE hash IN (...)`: a Buffer[] IN would return the correct rows
+ * in theory but leans on libsql's BLOB-parameter round-trip behaviour,
+ * while string values compare reliably. The cache key still includes the
+ * hash, so hash-distinct entries stay disambiguated.
+ * @param trx - Knex instance.
+ * @param entries - Decomposed entries.
+ * @param cache - `hexHash|value → id` cache; mutated in place.
+ * @returns Parallel array of value ids (same order as `entries`).
+ */
+async function resolveValueIds(trx, entries, cache) {
+    const hashByEntryIndex = entries.map((entry) => computeContentHash(entry.value));
+    const missing = [];
+    const missingKeys = new Set();
+    for (const [i, entry] of entries.entries()) {
+        const hash = hashByEntryIndex[i];
+        const key = headerValueCacheKey(hash, entry.value);
+        if (!cache.has(key) && !missingKeys.has(key)) {
+            missing.push({ key, hash, value: entry.value });
+            missingKeys.add(key);
+        }
+    }
+    // Chunked like the entries insert: knex compiles a multi-row
+    // insert-with-onConflict into a compound SELECT, and SQLite rejects
+    // more than 500 compound terms per statement.
+    for (let index = 0; index < missing.length; index += ENTRY_INSERT_CHUNK_SIZE) {
+        const chunk = missing.slice(index, index + ENTRY_INSERT_CHUNK_SIZE);
+        await trx('header_value_refs')
+            .insert(chunk.map(({ hash, value }) => ({ hash, value })))
+            .onConflict(['hash', 'value'])
+            .ignore();
+        const inserted = await trx('header_value_refs')
+            .select('id', 'hash', 'value')
+            .whereIn('value', chunk.map((m) => m.value));
+        for (const row of inserted) {
+            cache.set(headerValueCacheKey(Buffer.from(row.hash), row.value), row.id);
+        }
+    }
+    return entries.map((entry, i) => {
+        const key = headerValueCacheKey(hashByEntryIndex[i], entry.value);
+        const id = cache.get(key);
+        if (id === undefined) {
+            throw new Error(`upsertOneHeaderSet: value_id not resolved for ${entry.value.slice(0, 40)}…`);
+        }
+        return id;
+    });
+}

package/lib/archive/populate-ref-tables/volatile-header-names.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Header names — always lower-cased — that are EXCLUDED from `stable_hash`.
+ * These change on every response (timestamps / request IDs / cache
+ * telemetry) and would defeat the dedup goal if hashed alongside the
+ * stable set.
+ *
+ * `set-cookie` sits here because per-session cookies rotate constantly on
+ * authenticated sites — it is deliberately classified as volatile even
+ * though it is technically a stable security signal.
+ *
+ * Any header not in this set is
+ * treated as **stable** by {@link ./header-stability.ts:isVolatileHeader}
+ * — the safer default because a genuinely volatile header misclassified
+ * as stable only weakens dedup (its per-response values fork
+ * otherwise-identical stable profiles into separate `stable_hash`
+ * clusters), while a stable header misclassified as volatile would drop
+ * a meaningful signal from `stable_hash` and falsely cluster distinct
+ * header configurations together.
+ */
+export declare const VOLATILE_HEADER_NAMES: ReadonlySet<string>;