@nitpicker/archive 0.0.0-stage → 0.25.0

package/lib/populate-ref-tables/populate-text-refs.js

@@ -0,0 +1,133 @@
+import { createProgressReporter } from '../create-progress-reporter.js';
+import { computeContentHash } from './compute-content-hash.js';
+/**
+ * Number of `text_refs` rows sent per `INSERT ... VALUES (...)` statement.
+ * Each row binds 2 params (hash + text), so 500 rows = 1000 params — well
+ * under SQLite's default `SQLITE_MAX_VARIABLE_NUMBER`.
+ */
+const INSERT_CHUNK_SIZE = 500;
+/**
+ * Rows scanned per keyset-paginated `SELECT` chunk. One SELECT reads all
+ * text columns for the source table's chunk simultaneously so the pages
+ * table is scanned exactly once per populate call, not N-times-once-per-
+ * text-column.
+ */
+const READ_CHUNK_SIZE = 5000;
+/**
+ * Source table + its text-shaped columns. Groups by table so each table
+ * is scanned exactly once with a single SELECT that covers every
+ * relevant column — a per-column scan would run 8 separate `pages`
+ * scans and multiply migration wall-clock accordingly.
+ *
+ * `dom_path` is intentionally absent: dom_path strings are derived from
+ * the stored HTML blob, not from any column of the current write-model,
+ * so this scan has nothing to read. Upserting dom_path strings into
+ * `text_refs` is the job of the image-items populate
+ * (`populate-entity-tables/populate-image-items.ts`), not this pass.
+ */
+const TEXT_SOURCES = [
+    { table: 'anchors', columns: ['textContent'] },
+    { table: 'images', columns: ['alt'] },
+    {
+        table: 'pages',
+        columns: [
+            'title',
+            'description',
+            'keywords',
+            'robots_raw',
+            'og_title',
+            'og_description',
+            'twitter_title',
+            'twitter_description',
+        ],
+    },
+];
+/**
+ * Populates `text_refs` from every text-shaped column across `anchors`,
+ * `images`, and `pages` (issue #191).
+ *
+ * Rules:
+ *
+ * - **Content-hash dedup contract**: identical text produces one row. The
+ *   hash is derived by {@link computeContentHash} (currently SHA-256,
+ *   matching `page_html_blobs.hash`; see that function's docs for the
+ *   algorithm-choice rationale).
+ * - **Empty and null are skipped** — an empty `<a>` tag adds no text to
+ *   the dictionary. Anchors with no text still get an `anchor_edges` row
+ *   in 0.13, just with `first_text_id = NULL`.
+ * - **INSERT OR IGNORE on (hash, text)** — the composite UNIQUE from the
+ *   0.13 DDL — makes the step idempotent across partial-failure
+ *   restarts.
+ *
+ * Peak memory is bounded by the count of distinct texts (all texts are
+ * held in a Map before insert). On the reference archive this Map is
+ * about 5 MB of anchor textContent + a few MB of page meta text — well
+ * within the migration process budget.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of each source table scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateTextRefs(trx);
+ * });
+ */
+export async function populateTextRefs(trx, onProgress) {
+    const seen = new Map();
+    for (const source of TEXT_SOURCES) {
+        const hasTable = await trx.schema.hasTable(source.table);
+        if (!hasTable) {
+            continue;
+        }
+        const presentColumns = [];
+        for (const column of source.columns) {
+            if (await trx.schema.hasColumn(source.table, column)) {
+                presentColumns.push(column);
+            }
+        }
+        if (presentColumns.length === 0) {
+            continue;
+        }
+        const countRows = await trx(source.table).count({ n: '*' });
+        const total = Number(countRows[0]?.n ?? 0);
+        const report = createProgressReporter(`text_refs (${source.table})`, total, onProgress);
+        let processed = 0;
+        let cursor = 0;
+        while (true) {
+            const rows = await trx(source.table)
+                .select('id', ...presentColumns)
+                .where('id', '>', cursor)
+                .orderBy('id', 'asc')
+                .limit(READ_CHUNK_SIZE);
+            if (rows.length === 0) {
+                break;
+            }
+            cursor = rows.at(-1).id;
+            processed += rows.length;
+            report(processed);
+            for (const row of rows) {
+                for (const column of presentColumns) {
+                    const value = row[column];
+                    if (typeof value !== 'string' || value === '') {
+                        continue;
+                    }
+                    const hash = computeContentHash(value);
+                    const key = hash.toString('hex');
+                    if (!seen.has(key)) {
+                        seen.set(key, value);
+                    }
+                }
+            }
+        }
+    }
+    if (seen.size === 0) {
+        return;
+    }
+    const inserts = [];
+    for (const [hex, text] of seen) {
+        inserts.push({ hash: Buffer.from(hex, 'hex'), text });
+    }
+    for (let index = 0; index < inserts.length; index += INSERT_CHUNK_SIZE) {
+        const chunk = inserts.slice(index, index + INSERT_CHUNK_SIZE);
+        await trx('text_refs').insert(chunk).onConflict(['hash', 'text']).ignore();
+    }
+}

package/lib/populate-ref-tables/populate-url-refs.d.ts

@@ -0,0 +1,28 @@
+import type { ProgressCallback } from '../create-progress-reporter.js';
+import type { Knex } from 'knex';
+/**
+ * Populates `url_refs` from every URL-shaped column across `pages`,
+ * `resources`, and `images` (issue #191).
+ *
+ * Runs two passes per source table:
+ *
+ * 1. **Collect distinct URLs** into an in-process `Set<string>`, streaming
+ *    rows from each source table by keyset pagination on `id`. All the
+ *    table's URL columns are read in a single SELECT so the pages scan
+ *    is O(rows), not O(rows × columns). Peak memory is bounded by the
+ *    count of distinct URLs, not the row count.
+ * 2. **Bulk-insert** into `url_refs` in chunks of {@link INSERT_CHUNK_SIZE},
+ *    with the decomposed columns (`scheme` / `host` / `port` / `path` /
+ *    `query_hash` / `fragment`) derived in JS by {@link decomposeUrl}.
+ *
+ * `INSERT OR IGNORE` on `url_refs.url` makes the step idempotent —
+ * repeated invocations after partial failure only add new URLs.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of each source table scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateUrlRefs(trx);
+ * });
+ */
+export declare function populateUrlRefs(trx: Knex, onProgress?: ProgressCallback): Promise<void>;

package/lib/populate-ref-tables/populate-url-refs.js

@@ -0,0 +1,148 @@
+import { createProgressReporter } from '../create-progress-reporter.js';
+import { DATA_URI_URL_REFS_LIMIT } from './data-uri-url-refs-limit.js';
+import { decomposeUrl } from './decompose-url.js';
+/**
+ * Rows inserted per `INSERT INTO url_refs ... VALUES (...)` statement. Kept
+ * well under SQLite's `SQLITE_MAX_VARIABLE_NUMBER` (default 32766 on
+ * modern builds) — each row binds 7 parameters (url + scheme + host +
+ * port + path + query_hash + fragment), so 500 rows = 3500 params.
+ */
+const INSERT_CHUNK_SIZE = 500;
+/**
+ * Rows scanned per keyset-paginated `SELECT` chunk. One SELECT reads all
+ * URL columns for the source table's chunk simultaneously so the pages
+ * table is scanned exactly once per populate call, not N-times-once-
+ * per-URL-column — a per-column scan would multiply migration wall-clock
+ * by the count of URL-shaped columns.
+ */
+const READ_CHUNK_SIZE = 5000;
+/**
+ * Source table + its URL-shaped columns. Each column is scanned in
+ * lock-step per row, then filtered through
+ * {@link routeDataUriAwayFromUrlRefs} so large data URIs skip `url_refs`
+ * and route to `blob_refs` instead (handled by
+ * {@link ./populate-blob-refs.ts}).
+ *
+ * The filter is applied to EVERY URL column (not just image columns):
+ * `og:image`, `icon_href`, `apple-touch-icon`, `twitter:image`, and even
+ * `og:url` / `canonical` can legally hold data URIs in the wild, and a
+ * >512-byte data URI in any of those must land in `blob_refs`, not
+ * pollute the URL dictionary.
+ */
+const URL_SOURCES = [
+    {
+        table: 'pages',
+        columns: [
+            'url',
+            'canonical',
+            'og_url',
+            'og_image',
+            'icon_href',
+            'appleTouchIcon_href',
+            'amphtml',
+            'manifest',
+            'twitter_image',
+        ],
+    },
+    { table: 'resources', columns: ['url'] },
+    { table: 'images', columns: ['src', 'currentSrc'] },
+];
+/**
+ * Predicate applied to every URL column value: keep the value in the
+ * `url_refs` stream unless it is a data URI larger than
+ * {@link DATA_URI_URL_REFS_LIMIT}. Large data URIs route to `blob_refs`
+ * instead (via {@link ./populate-blob-refs.ts}).
+ * @param value - Raw column value (non-null, non-empty by contract).
+ * @returns `true` if the value belongs in `url_refs`.
+ */
+function routeDataUriAwayFromUrlRefs(value) {
+    if (!value.startsWith('data:')) {
+        return true;
+    }
+    return value.length <= DATA_URI_URL_REFS_LIMIT;
+}
+/**
+ * Populates `url_refs` from every URL-shaped column across `pages`,
+ * `resources`, and `images` (issue #191).
+ *
+ * Runs two passes per source table:
+ *
+ * 1. **Collect distinct URLs** into an in-process `Set<string>`, streaming
+ *    rows from each source table by keyset pagination on `id`. All the
+ *    table's URL columns are read in a single SELECT so the pages scan
+ *    is O(rows), not O(rows × columns). Peak memory is bounded by the
+ *    count of distinct URLs, not the row count.
+ * 2. **Bulk-insert** into `url_refs` in chunks of {@link INSERT_CHUNK_SIZE},
+ *    with the decomposed columns (`scheme` / `host` / `port` / `path` /
+ *    `query_hash` / `fragment`) derived in JS by {@link decomposeUrl}.
+ *
+ * `INSERT OR IGNORE` on `url_refs.url` makes the step idempotent —
+ * repeated invocations after partial failure only add new URLs.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of each source table scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateUrlRefs(trx);
+ * });
+ */
+export async function populateUrlRefs(trx, onProgress) {
+    const seen = new Set();
+    for (const source of URL_SOURCES) {
+        const hasTable = await trx.schema.hasTable(source.table);
+        if (!hasTable) {
+            continue;
+        }
+        const presentColumns = [];
+        for (const column of source.columns) {
+            if (await trx.schema.hasColumn(source.table, column)) {
+                presentColumns.push(column);
+            }
+        }
+        if (presentColumns.length === 0) {
+            continue;
+        }
+        const countRows = await trx(source.table).count({ n: '*' });
+        const total = Number(countRows[0]?.n ?? 0);
+        const report = createProgressReporter(`url_refs (${source.table})`, total, onProgress);
+        let processed = 0;
+        let cursor = 0;
+        while (true) {
+            const rows = await trx(source.table)
+                .select('id', ...presentColumns)
+                .where('id', '>', cursor)
+                .orderBy('id', 'asc')
+                .limit(READ_CHUNK_SIZE);
+            if (rows.length === 0) {
+                break;
+            }
+            cursor = rows.at(-1).id;
+            processed += rows.length;
+            report(processed);
+            for (const row of rows) {
+                for (const column of presentColumns) {
+                    const value = row[column];
+                    if (typeof value !== 'string' || value === '') {
+                        continue;
+                    }
+                    if (!routeDataUriAwayFromUrlRefs(value)) {
+                        continue;
+                    }
+                    seen.add(value);
+                }
+            }
+        }
+    }
+    if (seen.size === 0) {
+        return;
+    }
+    const inserts = [];
+    for (const url of seen) {
+        const decomposed = decomposeUrl(url);
+        inserts.push({ url, ...decomposed });
+    }
+    for (let index = 0; index < inserts.length; index += INSERT_CHUNK_SIZE) {
+        const chunk = inserts.slice(index, index + INSERT_CHUNK_SIZE);
+        await trx('url_refs').insert(chunk).onConflict('url').ignore();
+    }
+}

package/lib/populate-ref-tables/test-utils/count-rows.d.ts

@@ -0,0 +1,15 @@
+import type { Knex } from 'knex';
+/**
+ * Runs `SELECT count(?) FROM <table>` and returns the count as a plain
+ * JS number. Exists purely to keep spec files inside populate-ref-tables/ compliant
+ * with `unicorn/no-await-expression-member` — inlining
+ * `Number((await db(t).count(...))[0]!.n)` reads fine but violates the
+ * rule at every callsite.
+ * @param db - Knex instance (typically a spec-local in-memory DB).
+ * @param table - Table name.
+ * @param countColumn - Column to count; defaults to `'id'` because every
+ *   populate-refs table except `header_set_entries` / `header_flags` uses
+ *   that PK name. Pass `'header_set_id'` for the two exceptions.
+ * @returns Row count.
+ */
+export declare function countRows(db: Knex, table: string, countColumn?: string): Promise<number>;

package/lib/populate-ref-tables/test-utils/count-rows.js

@@ -0,0 +1,17 @@
+/**
+ * Runs `SELECT count(?) FROM <table>` and returns the count as a plain
+ * JS number. Exists purely to keep spec files inside populate-ref-tables/ compliant
+ * with `unicorn/no-await-expression-member` — inlining
+ * `Number((await db(t).count(...))[0]!.n)` reads fine but violates the
+ * rule at every callsite.
+ * @param db - Knex instance (typically a spec-local in-memory DB).
+ * @param table - Table name.
+ * @param countColumn - Column to count; defaults to `'id'` because every
+ *   populate-refs table except `header_set_entries` / `header_flags` uses
+ *   that PK name. Pass `'header_set_id'` for the two exceptions.
+ * @returns Row count.
+ */
+export async function countRows(db, table, countColumn = 'id') {
+    const rows = await db(table).count({ n: countColumn });
+    return Number(rows[0].n);
+}

package/lib/populate-ref-tables/types.d.ts

@@ -0,0 +1,197 @@
+/**
+ * Domain types for the 0.13 population step (issue #191). Every
+ * shared interface / type alias lives here per the repo-wide
+ * `型は types.ts に集約` rule; implementation modules export functions
+ * only.
+ */
+/**
+ * Decomposed URL components that populate `url_refs.{scheme,host,port,path,
+ * query_hash,fragment}`. All slots are populated in a single pass over the
+ * `url` string so callers do not have to re-parse per column.
+ */
+export interface DecomposedUrl {
+    /**
+     * URL scheme without the trailing colon (`https`, `http`, `mailto`, ...).
+     * `null` for URLs that fail `new URL(...)` parsing.
+     */
+    scheme: string | null;
+    /**
+     * Host lower-cased (`www.example.com`). `null` when the URL has no
+     * `authority` component (e.g. `mailto:`, `data:`, `javascript:`), and
+     * `null` on parse failure.
+     */
+    host: string | null;
+    /**
+     * TCP port when explicit in the URL; `null` when the URL uses the
+     * scheme default (WHATWG URL normalises default ports away — we do NOT
+     * synthesise a default here because two rows differing only by explicit
+     * vs implicit `:443` should hash to the same URL, and `url_refs.url` is
+     * the natural key, not this column).
+     */
+    port: number | null;
+    /**
+     * Path portion (`/foo/bar`). `null` when the URL has no path or the
+     * URL uses an opaque scheme whose "pathname" is really an in-band
+     * payload (`data:`, `blob:`, `javascript:`) — see
+     * {@link ./decompose-url.ts} for the opaque-scheme handling.
+     */
+    path: string | null;
+    /**
+     * 32-byte content hash of the raw query string exactly as it appears in
+     * the URL (`?a=1&b=2` — the leading `?` is stripped before hashing).
+     * `null` when the URL has no query string, when the query string is
+     * empty, or on parse failure.
+     *
+     * Storing the raw query would defeat dedup on tracker URLs whose
+     * per-request query keys blow up dictionary size; the hash retains the
+     * same-URL-⇒-same-row property without the payload.
+     */
+    query_hash: Buffer | null;
+    /**
+     * Fragment (`#section-2`) with the leading `#` stripped. `null` when
+     * the URL has no fragment or on parse failure.
+     */
+    fragment: string | null;
+}
+/**
+ * Successful decode of a `data:` URI: the raw payload bytes.
+ */
+export interface DecodedDataUri {
+    /** Payload bytes after base64 or percent-decoding. */
+    bytes: Buffer;
+}
+/**
+ * One header entry after decomposition — flattened out of the parsed JSON
+ * `Record<name, string | string[]>` so that:
+ *
+ * - Multiple values for the same name (e.g. two `Set-Cookie` lines) each
+ *   get their own entry with a distinct `occurrence` ordinal starting at 1.
+ * - `isVolatile` is pre-computed once per entry (rather than re-looked-up
+ *   at hash time) so both `stableHash` and the `header_set_entries`
+ *   inserts see identical classification.
+ */
+export interface HeaderEntry {
+    /** Lower-cased header name; comparison and dedup keys always use this form. */
+    name: string;
+    /** Raw header value (each element of `string[]` inputs becomes one entry). */
+    value: string;
+    /**
+     * 1-based ordinal among entries sharing the same `name` within one
+     * response. Always `1` for single-value headers; ≥ 2 when a name
+     * repeats (e.g. multiple `Set-Cookie` lines).
+     */
+    occurrence: number;
+    /** Pre-computed volatility. */
+    isVolatile: boolean;
+}
+/**
+ * Result of decomposing one raw `responseHeaders` JSON string into the
+ * shape required for `header_sets`, `header_set_entries`, and
+ * `header_flags` inserts. `null` results from the decompose helper
+ * indicate "no header set to insert" (null / `{}` / parse failure);
+ * callers write `header_set_id = null` to the referring row.
+ */
+export interface DecomposedHeaderSet {
+    /**
+     * 32-byte hash of the raw JSON string exactly as stored on the source
+     * row. Populates `header_sets.raw_json_hash` — a temporary column used
+     * by the 0.13 lookup to map an old `pages.responseHeaders` /
+     * `resources.responseHeaders` value back to its `header_sets.id`
+     * without any SQL function call.
+     */
+    rawJsonHash: Buffer;
+    /**
+     * 32-byte hash of the sorted `name=value` pairs of **all** entries
+     * (stable + volatile). Populates `header_sets.raw_hash` — a UNIQUE
+     * column so a JS-side upsert can dedup on identical decoded sets.
+     */
+    rawHash: Buffer;
+    /**
+     * 32-byte hash of the sorted `name=value` pairs of stable entries
+     * only. Populates `header_sets.stable_hash` — indexed so 0.13
+     * readers can answer "how many responses share this stable header
+     * profile" without scanning every entry.
+     */
+    stableHash: Buffer;
+    /**
+     * 32-byte hash of the sorted `name=value` pairs of volatile entries
+     * only, or `null` when the set has no volatile entries. Populates
+     * `header_sets.volatile_hash`.
+     */
+    volatileHash: Buffer | null;
+    /** Every decomposed entry, ordered by (name, occurrence). */
+    entries: readonly HeaderEntry[];
+    /** Total entry count — matches `header_sets.entry_count`. */
+    entryCount: number;
+    /**
+     * Number of stable entries — matches `header_sets.stable_entry_count`.
+     * `volatile_entry_count` is derived as `entry_count - stable_entry_count`
+     * and not stored separately.
+     */
+    stableEntryCount: number;
+}
+/**
+ * Shape written into `header_flags` for one `header_set_id`. `has_*`
+ * columns are `INTEGER NOT NULL` (0/1) per the 0.13 DDL;
+ * `cache_policy` is nullable and holds a compact summary of the
+ * `Cache-Control` value (or `null` when absent).
+ */
+export interface HeaderFlagsRow {
+    /** `1` when at least one entry's name equals `content-security-policy`. */
+    has_csp: 0 | 1;
+    /** `1` when at least one entry's name equals `x-frame-options`. */
+    has_x_frame_options: 0 | 1;
+    /** `1` when at least one entry's name equals `x-content-type-options`. */
+    has_x_content_type_options: 0 | 1;
+    /** `1` when at least one entry's name equals `strict-transport-security`. */
+    has_hsts: 0 | 1;
+    /** `1` when at least one entry's name equals `referrer-policy`. */
+    has_referrer_policy: 0 | 1;
+    /** `1` when at least one entry's name equals `permissions-policy`. */
+    has_permissions_policy: 0 | 1;
+    /** `1` when at least one entry's name equals `set-cookie`. */
+    has_set_cookie: 0 | 1;
+    /**
+     * The `cache-control` value, verbatim, when present; `null` when
+     * absent. Multi-value `Cache-Control` headers are joined with `, ` so
+     * the summary captures every directive without losing per-occurrence
+     * context.
+     */
+    cache_policy: string | null;
+}
+/**
+ * In-process caches shared by every header-table upsert against one
+ * archive connection. The three id-maps resolve `header_name_refs.id` /
+ * `header_value_refs.id` / `header_sets.id` without hitting the DB once
+ * a value has been seen (or preloaded from existing rows).
+ *
+ * `setIdsProcessedThisRun` tracks the subset of `setIdByRawJsonHash` /
+ * `setIdByRawHash` that the current process has already written
+ * entries + flags for. A preloaded id from a prior run is present in the
+ * id caches but NOT in this set — so the entries + flags inserts still
+ * run once (safe under `INSERT OR IGNORE`) to repair any missing rows
+ * left by a crashed writer. Once entries + flags for a given setId have
+ * been written this run, the shortcut skips redundant no-op inserts.
+ */
+export interface HeaderTableCaches {
+    /** `lower-cased name → header_name_refs.id`. */
+    readonly nameIdCache: Map<string, number>;
+    /** `hexHash|value → header_value_refs.id`. */
+    readonly valueIdCache: Map<string, number>;
+    /** `hex(raw_json_hash) → header_sets.id`. */
+    readonly setIdByRawJsonHash: Map<string, number>;
+    /** `hex(raw_hash) → header_sets.id`. */
+    readonly setIdByRawHash: Map<string, number>;
+    /** `header_sets.id` values already fully processed by this process. */
+    readonly setIdsProcessedThisRun: Set<number>;
+}
+/**
+ * Category label attached to each content-type rule. Mirrors the union
+ * `ContentTypeCategory` in `@nitpicker/query/src/types.ts` — kept as a
+ * plain string literal here so `@nitpicker/archive`'s 0.13 step
+ * does not need to depend on `@nitpicker/query` (which itself depends
+ * on this package). If the two copies drift, `classify-content-type.spec.ts`
+ * here and `content-type-rules.spec.ts` on the query side will disagree
+ * on the same fixture.
+ */
+export type ContentTypeCategory = 'html' | 'pdf' | 'csv' | 'word' | 'excel' | 'powerpoint' | 'image' | 'audio' | 'video' | 'font' | 'css' | 'javascript' | 'json' | 'xml' | 'archive' | 'text' | 'other' | 'unknown';

package/lib/populate-ref-tables/types.js

@@ -0,0 +1,7 @@
+/**
+ * Domain types for the 0.13 population step (issue #191). Every
+ * shared interface / type alias lives here per the repo-wide
+ * `型は types.ts に集約` rule; implementation modules export functions
+ * only.
+ */
+export {};

package/lib/populate-ref-tables/upsert-one-header-set.d.ts

@@ -0,0 +1,34 @@
+import type { DecomposedHeaderSet, HeaderTableCaches } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Upserts one decomposed header set and its entries + flags, returning
+ * the `header_sets.id`. Shared by the bulk archive populate
+ * ({@link ./populate-header-tables.ts}) and the crawler's per-response
+ * write path, so both produce byte-identical header rows for the same
+ * raw `responseHeaders` value.
+ *
+ * Set-id resolution order:
+ *
+ * 1. `raw_json_hash` cache — same JSON key ordering as a previous row.
+ * 2. `raw_hash` cache — same sorted-entries identity, different JSON key
+ *    ordering. This branch links the new `raw_json_hash` to the existing
+ *    setId in the in-process map so subsequent same-JSON hits fast-path.
+ * 3. Otherwise INSERT a new `header_sets` row.
+ *
+ * Entries + flags are inserted once per `setId` per process (guarded by
+ * `setIdsProcessedThisRun`); already-inserted rows short-circuit
+ * cheaply, and preloaded-but-unprocessed setIds (partial prior run) get
+ * their entries / flags repaired via `INSERT OR IGNORE`.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param decomposed - Result of `decomposeHeaderSet` (never `null` — the
+ *   caller short-circuits null decompositions to `header_set_id = null`).
+ * @param caches - The in-process caches, mutated in place. Create via
+ *   {@link ./create-header-table-caches.ts}.
+ * @returns The `header_sets.id` for this set.
+ * @example
+ * const decomposed = decomposeHeaderSet(rawJson);
+ * if (decomposed !== null) {
+ *   const setId = await upsertOneHeaderSet(knex, decomposed, caches);
+ * }
+ */
+export declare function upsertOneHeaderSet(trx: Knex, decomposed: DecomposedHeaderSet, caches: HeaderTableCaches): Promise<number>;