@nitpicker/crawler 0.11.0 → 0.13.0

package/lib/archive/create-entity-tables.js

@@ -0,0 +1,318 @@
+/**
+ * Creates the 6 core entity and edge tables of the 0.13 format (issue #192).
+ *
+ * These are the normalised entity / edge tables that replace the legacy
+ * write model (`pages`, `anchors`, `images`, `resources`, `resources-referrers`)
+ * in the 0.13 format. This migration is purely additive DDL — no existing table
+ * is modified and no row is inserted; the populate step
+ * (`populateEntityTables`) fills the tables from the legacy sources.
+ *
+ * Tables created:
+ *
+ * - `content_items` — replaces `pages` (core columns). The remaining
+ *   per-page metadata splits into `page_meta`.
+ * - `page_meta` — page-specific metadata (title, description, meta tags,
+ *   OpenGraph, Twitter, robots, denormalised aggregates, meta_extras JSON
+ *   pointer). Split from `pages` so that filter / cursor queries against
+ *   `content_items` scan a narrower row.
+ * - `resource_items` — replaces `resources`. Identity URL splits into
+ *   `url_id` (regular URLs) vs `url_blob_id` (large `data:` URIs, routed
+ *   via `blob_refs`) — a resource's own URL is captured verbatim from
+ *   the network layer and, unlike a page's URL, can legally be a large
+ *   inline `data:` URI (e.g. a CSS `background-image` sub-resource).
+ * - `anchor_edges` — replaces `anchors`. Deduped to distinct
+ *   `(page_id, href_page_id)` pairs; `count` records how many instances
+ *   were observed and `first_hash` / `first_text_id` capture the first
+ *   instance's identity so the most important observable fact per link is
+ *   preserved without every redundant row.
+ * - `resource_ref_edges` — replaces `resources-referrers`. Structural
+ *   rename with an added `count` column; `WITHOUT ROWID` because the
+ *   composite PK `(resource_id, page_id)` is the natural clustering key
+ *   and every column is small — the same reasoning as `page_html_blobs`.
+ * - `image_items` — replaces `images`. `src` / `currentSrc` split into
+ *   `*_url_id` (regular URLs) vs `*_blob_id` (large `data:` URIs, routed
+ *   via `blob_refs`); `sourceCode` is replaced by `dom_path_text_id`,
+ *   which the populate step derives from the `outerHTML` string.
+ *
+ * ### Key invariants
+ *
+ * **Same PK values as legacy.** `content_items.id`, `resource_items.id`,
+ * and `image_items.id` reuse the exact PK values from `pages.id`,
+ * `resources.id`, and `images.id` respectively. The populate step inserts
+ * rows with explicit IDs so every existing FK reference in `page_errors`,
+ * `page_tags`, `page_jsonld`, `page_html_ref`, and `resources-referrers`
+ * (before it becomes `resource_ref_edges`) survives the switch without
+ * any per-row UPDATE. `AUTOINCREMENT` is nevertheless applied on all four
+ * entity PKs (`content_items` / `resource_items` / `anchor_edges` /
+ * `image_items`) to match the legacy contract (`t.increments()` on
+ * `pages` / `resources` / `anchors` / `images`) — this prevents rowid
+ * reuse after DELETE, so external systems (analysis outputs, viewer
+ * bookmarks, Sheets exports) keyed on `content_item.id` cannot be
+ * silently reassigned to a different record.
+ *
+ * **UNIQUE(url_id) on content_items and resource_items.** The legacy
+ * `pages.url UNIQUE` / `resources.url UNIQUE` guaranteed one row per
+ * URL string. Under the new model the URL string is stored once in
+ * `url_refs` and each entity references `url_refs.id` via `url_id` —
+ * but `url_refs.url UNIQUE` only guarantees "one URL string → one
+ * ref id", it does NOT prevent two entity rows from claiming the same
+ * ref id. `UNIQUE(url_id)` on both `content_items` and `resource_items`
+ * re-establishes that guarantee at the entity level — without it a
+ * populate bug could attach two entity rows to the same URL, and every
+ * by-URL reader would silently return an arbitrary one. The UNIQUE
+ * constraint's auto-index also serves the by-URL seek path, so no
+ * separate `CREATE INDEX ... ON content_items(url_id)` is needed.
+ * `resource_items.url_id` is nullable (unlike `content_items.url_id`)
+ * because a resource's identity URL can itself be blob-routed — see the
+ * `resource_items url / blob mutual-exclusion CHECK` invariant below.
+ * `UNIQUE` on a nullable SQLite column only dedups the non-NULL values,
+ * so the guarantee still holds for every resource that does have a
+ * `url_id`.
+ *
+ * **`redirect_dest_id` self-reference is DEFERRABLE INITIALLY DEFERRED.**
+ * `content_items.redirect_dest_id` references `content_items(id)` (not
+ * `pages(id)`), mirroring the legacy `pages.redirectDestId → pages.id`
+ * self-reference. In the real archive shape, a redirect source can be
+ * discovered as an anchor (assigned a low `pages.id`) BEFORE its
+ * destination is crawled (higher `pages.id`), so a straight
+ * `ORDER BY id ASC` populate would violate the FK on the source row.
+ * Making the FK deferred lets the populate step commit all `content_items`
+ * inserts inside one transaction without any per-row ordering
+ * discipline — SQLite validates the constraint only at COMMIT time.
+ *
+ * **`page_meta.page_id` PK == FK ON DELETE CASCADE.** `page_meta` uses
+ * `page_id` as both the primary key and the FK back to `content_items(id)`.
+ * This is a 1:1 relation (every content_item has at most one meta row),
+ * so a separate autoincrement id would only bloat the row. The cascade
+ * mirrors the legacy `page_html_ref` / `page_tags` / `page_jsonld` FKs
+ * to `pages(id)` so a future cleanup pass can DELETE from
+ * `content_items` without leaving orphan meta rows.
+ *
+ * **`content_items.source` and `resource_items.source` DEFAULT 'crawled'.**
+ * The legacy `pages.source` / `resources.source` columns were
+ * `NOT NULL DEFAULT 'crawled'`, and multiple existing writer paths rely
+ * on the default (redirect-source insert, per-chunk resource insert, and
+ * `markBrowserScrape`). Preserving the default keeps those callsites
+ * working when the 0.13 write path re-points them at the new tables — dropping
+ * the default would turn a routine crawl into an immediate hard failure
+ * the first time a callsite omits `source`.
+ *
+ * **`anchor_edges` dedup shape.** `unique(page_id, href_page_id)` enforces
+ * the collapse from ≈ 13M legacy `anchors` rows to ≈ 9.7M `anchor_edges`
+ * rows (~25 % reduction on the reference archive).
+ * `first_hash` and `first_text_id` capture the anchor `hash` and
+ * `textContent` of the FIRST instance encountered (lowest `anchors.id`
+ * for the pair); duplicate observations are counted via `count` without
+ * retaining their bodies. The UNIQUE constraint's auto-index on
+ * `(page_id, href_page_id)` also serves any `WHERE page_id = ?` seek via
+ * SQLite's leading-prefix rule, so no separate `CREATE INDEX ... ON
+ * anchor_edges(page_id)` is needed; only the reverse-direction
+ * `idx_anchor_edges_href` on `href_page_id` alone is added.
+ *
+ * **`image_items.dom_path_text_id NOT NULL`.** Every image row must
+ * resolve to a `text_refs` entry describing its DOM position. The
+ * image-items populate derives the path from `images.sourceCode` (the
+ * stored `outerHTML`); an
+ * image whose `sourceCode` is empty falls back to the synthetic label
+ * `img[unknown]` (via the image-items populate) so the FK is never violated.
+ *
+ * **`image_items` src / blob mutual-exclusion CHECK.** `src_url_id` and
+ * `src_blob_id` (and their `current_src_*` mirror) are separate nullable
+ * columns because a `<img>` value can be either a plain URL or a large
+ * `data:` URI. A single legacy `images.src` TEXT column could not
+ * express the ambiguity; the new schema splits the two but adds a CHECK
+ * so at most one slot per (src, currentSrc) pair is non-null. Without
+ * the CHECK a populate bug or future writer could silently write both,
+ * and the viewer would arbitrarily pick one to render.
+ *
+ * **`resource_items` url / blob mutual-exclusion CHECK.** Same rationale
+ * as the image CHECK above, but stricter: `(url_id IS NULL) !=
+ * (url_blob_id IS NULL)` requires EXACTLY one slot to be non-null (not
+ * "at most one") because a resource — unlike an `<img>` that can
+ * legitimately have neither `src` nor `currentSrc` — is always
+ * identified by some URL. `resolveUrlOrBlob` (writer) /
+ * `resolveUrlOrBlobFromMaps` (populate) are shared with `image_items`'
+ * routing so the two entities never disagree on which values count as
+ * "large data URI".
+ *
+ * ### Index rationale
+ *
+ * Every index below reflects a legacy-baseline single-column index that
+ * an existing hot path already relies on. `content_items` gets six
+ * indexes: `url_id` (via `UNIQUE` auto-index), `is_external`, `scraped`,
+ * `redirect_dest_id` (for the Page Detail inbound-redirect list),
+ * `content_type_id`, `crawl_order`, and `source` — all present on
+ * legacy `pages`. `page_meta` gets `og_type` and `robots_noindex`
+ * because both were indexed on legacy `pages` for the noindex /
+ * og:type filter hot paths; after the split those readers land on
+ * `page_meta` and need the indexes to remain sub-second. `resource_items` gets `url_id`
+ * (via `UNIQUE` auto-index) and `source`. `anchor_edges` gets one
+ * reverse-direction index (`idx_anchor_edges_href`); the UNIQUE
+ * auto-index covers the forward direction. `resource_ref_edges` needs
+ * an explicit reverse-direction index on `page_id` because the
+ * composite PK `(resource_id, page_id)` only satisfies the `resource_id`-
+ * first prefix; the legacy `resources-referrers` table had exactly
+ * this reverse-direction index. `image_items` gets one index on
+ * `page_id`. No summary / listfilter compound indexes are created here —
+ * they belong to the viewer read model built directly against these
+ * tables.
+ *
+ * ### Idempotency
+ *
+ * All statements use `CREATE TABLE IF NOT EXISTS` / `CREATE INDEX IF NOT
+ * EXISTS` so the primitive is safe to call multiple times against the
+ * same DB. Both call sites (`initSchema` and
+ * `migrateEntityTables`) still guard on their own sentinels,
+ * but the function itself no longer depends on the sentinel being
+ * checked first — this closes the partial-corruption gap where, say,
+ * `content_items` exists but `page_meta` was dropped by an external
+ * repair pass.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function createEntityTables(instance) {
+    await instance.raw(`
+		CREATE TABLE IF NOT EXISTS content_items (
+			id               INTEGER PRIMARY KEY AUTOINCREMENT,
+			url_id           INTEGER NOT NULL UNIQUE REFERENCES url_refs(id),
+			is_external      INTEGER NOT NULL,
+			scraped          INTEGER NOT NULL,
+			is_target        INTEGER NOT NULL,
+			status           INTEGER,
+			status_text      TEXT,
+			content_type_id  INTEGER REFERENCES content_type_refs(id),
+			content_length   INTEGER,
+			header_set_id    INTEGER REFERENCES header_sets(id),
+			redirect_dest_id INTEGER REFERENCES content_items(id) DEFERRABLE INITIALLY DEFERRED,
+			source           TEXT NOT NULL DEFAULT 'crawled',
+			first_crawled_at INTEGER,
+			last_crawled_at  INTEGER,
+			crawl_order      INTEGER,
+			is_skipped       INTEGER,
+			skip_reason      TEXT
+		)
+	`);
+    await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_external ON content_items(is_external)');
+    await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_scraped ON content_items(scraped)');
+    await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_redirect_dest_id ON content_items(redirect_dest_id)');
+    await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_content_type_id ON content_items(content_type_id)');
+    await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_crawl_order ON content_items(crawl_order)');
+    await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_source ON content_items(source)');
+    await instance.raw(`
+		CREATE TABLE IF NOT EXISTS page_meta (
+			page_id                     INTEGER PRIMARY KEY REFERENCES content_items(id) ON DELETE CASCADE,
+			lang                        TEXT,
+			dir                         TEXT,
+			charset                     TEXT,
+			base_href                   TEXT,
+			viewport_raw                TEXT,
+			theme_color                 TEXT,
+			application_name            TEXT,
+			author                      TEXT,
+			generator                   TEXT,
+			publisher                   TEXT,
+			title_text_id               INTEGER REFERENCES text_refs(id),
+			description_text_id         INTEGER REFERENCES text_refs(id),
+			keywords_text_id            INTEGER REFERENCES text_refs(id),
+			robots_raw_text_id          INTEGER REFERENCES text_refs(id),
+			robots_noindex              INTEGER,
+			robots_nofollow             INTEGER,
+			robots_noarchive            INTEGER,
+			robots_noimageindex         INTEGER,
+			googlebot                   TEXT,
+			canonical_url_id            INTEGER REFERENCES url_refs(id),
+			amphtml_url_id              INTEGER REFERENCES url_refs(id),
+			manifest_url_id             INTEGER REFERENCES url_refs(id),
+			icon_url_id                 INTEGER REFERENCES url_refs(id),
+			apple_touch_icon_url_id     INTEGER REFERENCES url_refs(id),
+			og_type                     TEXT,
+			og_title_text_id            INTEGER REFERENCES text_refs(id),
+			og_description_text_id      INTEGER REFERENCES text_refs(id),
+			og_url_id                   INTEGER REFERENCES url_refs(id),
+			og_image_url_id             INTEGER REFERENCES url_refs(id),
+			og_site_name                TEXT,
+			og_image_alt                TEXT,
+			og_image_width              TEXT,
+			og_image_height             TEXT,
+			og_locale                   TEXT,
+			og_article_published_time   TEXT,
+			og_article_modified_time    TEXT,
+			twitter_card                TEXT,
+			twitter_site                TEXT,
+			twitter_creator             TEXT,
+			twitter_title_text_id       INTEGER REFERENCES text_refs(id),
+			twitter_description_text_id INTEGER REFERENCES text_refs(id),
+			twitter_image_url_id        INTEGER REFERENCES url_refs(id),
+			fb_app_id                   TEXT,
+			verification_google         TEXT,
+			format_detection_telephone  INTEGER,
+			tag_count                   INTEGER,
+			jsonld_count                INTEGER,
+			tags_providers_csv          TEXT,
+			meta_extras_json_id         INTEGER REFERENCES json_refs(id)
+		)
+	`);
+    await instance.raw('CREATE INDEX IF NOT EXISTS idx_page_meta_og_type ON page_meta(og_type)');
+    await instance.raw('CREATE INDEX IF NOT EXISTS idx_page_meta_robots_noindex ON page_meta(robots_noindex)');
+    await instance.raw(`
+		CREATE TABLE IF NOT EXISTS resource_items (
+			id               INTEGER PRIMARY KEY AUTOINCREMENT,
+			url_id           INTEGER UNIQUE REFERENCES url_refs(id),
+			url_blob_id      INTEGER UNIQUE REFERENCES blob_refs(id),
+			is_external      INTEGER NOT NULL,
+			status           INTEGER,
+			status_text      TEXT,
+			content_type_id  INTEGER REFERENCES content_type_refs(id),
+			content_length   INTEGER,
+			header_set_id    INTEGER REFERENCES header_sets(id),
+			compress         TEXT,
+			cdn              TEXT,
+			source           TEXT NOT NULL DEFAULT 'crawled',
+			CHECK ((url_id IS NULL) != (url_blob_id IS NULL))
+		)
+	`);
+    await instance.raw('CREATE INDEX IF NOT EXISTS idx_resource_items_source ON resource_items(source)');
+    await instance.raw(`
+		CREATE TABLE IF NOT EXISTS anchor_edges (
+			id            INTEGER PRIMARY KEY AUTOINCREMENT,
+			page_id       INTEGER NOT NULL REFERENCES content_items(id),
+			href_page_id  INTEGER NOT NULL REFERENCES content_items(id),
+			count         INTEGER NOT NULL,
+			first_hash    TEXT,
+			first_text_id INTEGER REFERENCES text_refs(id),
+			UNIQUE(page_id, href_page_id)
+		)
+	`);
+    await instance.raw('CREATE INDEX IF NOT EXISTS idx_anchor_edges_href ON anchor_edges(href_page_id)');
+    await instance.raw(`
+		CREATE TABLE IF NOT EXISTS resource_ref_edges (
+			resource_id INTEGER NOT NULL REFERENCES resource_items(id),
+			page_id     INTEGER NOT NULL REFERENCES content_items(id),
+			count       INTEGER NOT NULL DEFAULT 1,
+			PRIMARY KEY(resource_id, page_id)
+		) WITHOUT ROWID
+	`);
+    await instance.raw('CREATE INDEX IF NOT EXISTS idx_resource_ref_edges_page ON resource_ref_edges(page_id)');
+    await instance.raw(`
+		CREATE TABLE IF NOT EXISTS image_items (
+			id                  INTEGER PRIMARY KEY AUTOINCREMENT,
+			page_id             INTEGER NOT NULL REFERENCES content_items(id),
+			src_url_id          INTEGER REFERENCES url_refs(id),
+			current_src_url_id  INTEGER REFERENCES url_refs(id),
+			src_blob_id         INTEGER REFERENCES blob_refs(id),
+			current_src_blob_id INTEGER REFERENCES blob_refs(id),
+			alt_text_id         INTEGER REFERENCES text_refs(id),
+			width               REAL,
+			height              REAL,
+			natural_width       INTEGER,
+			natural_height      INTEGER,
+			is_lazy             INTEGER,
+			viewport_width      INTEGER,
+			dom_path_text_id    INTEGER NOT NULL REFERENCES text_refs(id),
+			CHECK (
+				(src_url_id IS NULL OR src_blob_id IS NULL)
+				AND (current_src_url_id IS NULL OR current_src_blob_id IS NULL)
+			)
+		)
+	`);
+    await instance.raw('CREATE INDEX IF NOT EXISTS idx_image_items_page ON image_items(page_id)');
+}

package/lib/archive/create-progress-reporter.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Emits a human-readable progress line for a long-running populate step.
+ * Injected the same way as `PageDomPathResolver`
+ * (`populate-entity-tables/populate-image-items.ts`) — the crawler
+ * package stays silent by default for library consumers, while
+ * `scripts/migrate-to-0.13.mjs` supplies a `console.log`-backed callback.
+ */
+export type ProgressCallback = (message: string) => void;
+/**
+ * Returns a `report(processed)` function that calls `onProgress` at most
+ * once per 5%-of-`total` tier crossed, so a keyset-paginated loop over a
+ * multi-million-row table can report progress every chunk without
+ * flooding the log with one line per chunk.
+ * @param label - Prefix identifying which populate step / source table
+ *   this reporter tracks, e.g. `"url_refs (resources)"`.
+ * @param total - Total row/page count the loop will scan, precomputed by
+ *   the caller via a `COUNT(*)`.
+ * @param onProgress - Caller-supplied sink, or `undefined` to disable
+ *   reporting entirely (zero overhead — `report` becomes a no-op).
+ * @returns `report(processed)` — call after each chunk with the
+ *   cumulative count processed so far.
+ * @example
+ * const total = Number((await trx('url_refs').count({ n: '*' }))[0]?.n ?? 0);
+ * const report = createProgressReporter('url_refs', total, onProgress);
+ * let processed = 0;
+ * // ...inside the read-chunk loop...
+ * processed += rows.length;
+ * report(processed);
+ */
+export declare function createProgressReporter(label: string, total: number, onProgress: ProgressCallback | undefined): (processed: number) => void;

package/lib/archive/create-progress-reporter.js

@@ -0,0 +1,38 @@
+/** Progress tiers per run — one report per 5% of `total`, so a 13M-row table produces ~20 lines, not one per chunk. */
+const TIERS = 20;
+/**
+ * Returns a `report(processed)` function that calls `onProgress` at most
+ * once per 5%-of-`total` tier crossed, so a keyset-paginated loop over a
+ * multi-million-row table can report progress every chunk without
+ * flooding the log with one line per chunk.
+ * @param label - Prefix identifying which populate step / source table
+ *   this reporter tracks, e.g. `"url_refs (resources)"`.
+ * @param total - Total row/page count the loop will scan, precomputed by
+ *   the caller via a `COUNT(*)`.
+ * @param onProgress - Caller-supplied sink, or `undefined` to disable
+ *   reporting entirely (zero overhead — `report` becomes a no-op).
+ * @returns `report(processed)` — call after each chunk with the
+ *   cumulative count processed so far.
+ * @example
+ * const total = Number((await trx('url_refs').count({ n: '*' }))[0]?.n ?? 0);
+ * const report = createProgressReporter('url_refs', total, onProgress);
+ * let processed = 0;
+ * // ...inside the read-chunk loop...
+ * processed += rows.length;
+ * report(processed);
+ */
+export function createProgressReporter(label, total, onProgress) {
+    let lastTier = 0;
+    return (processed) => {
+        if (onProgress === undefined || total === 0) {
+            return;
+        }
+        const tier = Math.floor((processed / total) * TIERS);
+        if (tier <= lastTier) {
+            return;
+        }
+        lastTier = tier;
+        const percent = Math.min(100, Math.round((processed / total) * 100));
+        onProgress(`${label}: ${processed}/${total} (${percent}%)`);
+    };
+}

package/lib/archive/create-ref-tables.d.ts

@@ -0,0 +1,35 @@
+import type { Knex } from 'knex';
+/**
+ * Creates the 10 dictionary tables of the 0.13 format (issue #190) and their index.
+ *
+ * These are the ref / header dictionary tables that form the durable
+ * write-model of the 0.13 format: `url_refs`, `content_type_refs`,
+ * `text_refs`, `json_refs`, `blob_refs`, `header_name_refs`,
+ * `header_value_refs`, `header_sets` (+ `idx_header_sets_stable`),
+ * `header_set_entries`, `header_flags`.
+ *
+ * The DDL is shared between fresh-archive provisioning ({@link initSchema}
+ * calls this on a brand new DB) and the lazy migration path
+ * ({@link migrateRefTables} calls it on archives created before the
+ * 0.13 format). Keeping the schema in one function guarantees that both
+ * origin points produce byte-identical tables — a divergence would silently
+ * break the 0.13 population step, which relies on the exact UNIQUE
+ * constraints and CHECK clauses declared here.
+ *
+ * Hash columns store 32-byte BLAKE3 as `BLOB`, matching `page_html_blobs.hash`.
+ * `blob_refs` intentionally uses a regular integer rowid PK (unlike
+ * `page_html_blobs`) because `image_items.src_blob_id` (added in 0.13)
+ * needs a plain integer FK and auto-increment is incompatible with WITHOUT
+ * ROWID; `UNIQUE(hash)` still enforces content-addressable dedup.
+ *
+ * `header_set_entries` is WITHOUT ROWID because the composite PK
+ * `(header_set_id, name_id, occurrence)` is the natural clustering key and
+ * every column is small — the same reasoning as `page_html_blobs`.
+ *
+ * `header_sets.raw_json_hash` is a temporary column used only by the
+ * legacy-to-0.13 population step for lookups against the legacy
+ * `responseHeaders` JSON; a follow-up cleanup migration is expected to
+ * drop it.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function createRefTables(instance: Knex): Promise<void>;

package/lib/archive/create-ref-tables.js

@@ -0,0 +1,188 @@
+/**
+ * Creates the 10 dictionary tables of the 0.13 format (issue #190) and their index.
+ *
+ * These are the ref / header dictionary tables that form the durable
+ * write-model of the 0.13 format: `url_refs`, `content_type_refs`,
+ * `text_refs`, `json_refs`, `blob_refs`, `header_name_refs`,
+ * `header_value_refs`, `header_sets` (+ `idx_header_sets_stable`),
+ * `header_set_entries`, `header_flags`.
+ *
+ * The DDL is shared between fresh-archive provisioning ({@link initSchema}
+ * calls this on a brand new DB) and the lazy migration path
+ * ({@link migrateRefTables} calls it on archives created before the
+ * 0.13 format). Keeping the schema in one function guarantees that both
+ * origin points produce byte-identical tables — a divergence would silently
+ * break the 0.13 population step, which relies on the exact UNIQUE
+ * constraints and CHECK clauses declared here.
+ *
+ * Hash columns store 32-byte BLAKE3 as `BLOB`, matching `page_html_blobs.hash`.
+ * `blob_refs` intentionally uses a regular integer rowid PK (unlike
+ * `page_html_blobs`) because `image_items.src_blob_id` (added in 0.13)
+ * needs a plain integer FK and auto-increment is incompatible with WITHOUT
+ * ROWID; `UNIQUE(hash)` still enforces content-addressable dedup.
+ *
+ * `header_set_entries` is WITHOUT ROWID because the composite PK
+ * `(header_set_id, name_id, occurrence)` is the natural clustering key and
+ * every column is small — the same reasoning as `page_html_blobs`.
+ *
+ * `header_sets.raw_json_hash` is a temporary column used only by the
+ * legacy-to-0.13 population step for lookups against the legacy
+ * `responseHeaders` JSON; a follow-up cleanup migration is expected to
+ * drop it.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function createRefTables(instance) {
+    // Unified URL dictionary. `url` is the natural unique key; scheme /
+    // host / port / path / query_hash / fragment are pre-decomposed columns
+    // filled by the 0.13 population step so filters like
+    // "same-host resources" avoid re-parsing the URL string per row.
+    // `query_hash` is BLAKE3 of the raw query string — storing query strings
+    // inline would defeat dedup on tracker URLs.
+    await instance.raw(`
+		CREATE TABLE url_refs (
+			id         INTEGER PRIMARY KEY,
+			url        TEXT NOT NULL UNIQUE,
+			scheme     TEXT,
+			host       TEXT,
+			port       INTEGER,
+			path       TEXT,
+			query_hash BLOB,
+			fragment   TEXT
+		)
+	`);
+    // Content-type dictionary. `raw` is the wire value (unique).
+    // `normalized` is the lowercase MIME with parameters stripped;
+    // `category` is the coarse bucket used by viewer filters ('html',
+    // 'image', ...). Populated in 0.13 by `classifyContentType`.
+    await instance.raw(`
+		CREATE TABLE content_type_refs (
+			id         INTEGER PRIMARY KEY,
+			raw        TEXT NOT NULL UNIQUE,
+			normalized TEXT NOT NULL,
+			category   TEXT NOT NULL
+		)
+	`);
+    // Short-text dictionary. Covers anchor textContent, image alt, page
+    // meta strings, and dom_path. The `(hash, text)` composite UNIQUE
+    // mirrors `analysis_text_refs` so lookups prefix-seek on `hash`. The
+    // two dictionaries stay separate because merging would require
+    // re-keying every existing `analysis_violations` row (out of scope
+    // for the 0.13 migration; revisit in a later cleanup pass).
+    await instance.raw(`
+		CREATE TABLE text_refs (
+			id   INTEGER PRIMARY KEY,
+			hash BLOB NOT NULL,
+			text TEXT NOT NULL,
+			UNIQUE(hash, text)
+		)
+	`);
+    // Large JSON payload dictionary. Covers `pages.meta_extras`. `hash` is
+    // BLAKE3 of the pre-compression JSON bytes; `codec` records whether
+    // `json_text` is zstd-compressed. Sizes are stored so consumers can
+    // estimate savings without decompressing.
+    await instance.raw(`
+		CREATE TABLE json_refs (
+			id          INTEGER PRIMARY KEY,
+			hash        BLOB NOT NULL UNIQUE,
+			json_text   BLOB NOT NULL,
+			codec       TEXT NOT NULL CHECK(codec IN ('zstd', 'none')),
+			size_raw    INTEGER NOT NULL,
+			size_stored INTEGER NOT NULL
+		)
+	`);
+    // Large binary payload dictionary. Covers `data:` URIs longer than
+    // 512 bytes attached to `<img>` src / currentSrc. Well-formed http(s)
+    // URLs virtually never exceed 512 chars, so this threshold routes all
+    // data URIs here without misclassifying URLs.
+    await instance.raw(`
+		CREATE TABLE blob_refs (
+			id          INTEGER PRIMARY KEY,
+			hash        BLOB NOT NULL UNIQUE,
+			body        BLOB NOT NULL,
+			codec       TEXT NOT NULL CHECK(codec IN ('zstd', 'none')),
+			size_raw    INTEGER NOT NULL,
+			size_stored INTEGER NOT NULL
+		)
+	`);
+    // Response-header decomposition tables. `responseHeaders` on the old
+    // `pages` / `resources` tables was a JSON blob whose raw hash barely
+    // deduped (99.4 % distinct on the reference archive) because volatile
+    // headers like `Date` / `ETag` / `CF-Ray` change every response. The
+    // five tables below split header sets into (name, value, occurrence)
+    // tuples so a stable-only hash can achieve real dedup, and expose the
+    // commonly-checked security flags as pre-computed booleans in
+    // `header_flags` so `checkHeaders` no longer needs a per-row LIKE
+    // scan.
+    await instance.raw(`
+		CREATE TABLE header_name_refs (
+			id   INTEGER PRIMARY KEY,
+			name TEXT NOT NULL UNIQUE
+		)
+	`);
+    await instance.raw(`
+		CREATE TABLE header_value_refs (
+			id    INTEGER PRIMARY KEY,
+			hash  BLOB NOT NULL,
+			value TEXT NOT NULL,
+			UNIQUE(hash, value)
+		)
+	`);
+    // `raw_json_hash` is BLAKE3 of the raw `responseHeaders` JSON string
+    // exactly as stored in the old tables — kept only so the 0.13
+    // migration can look up an existing `header_sets.id` without calling
+    // any SQL function (SQLite has no built-in BLAKE3). Scheduled for drop
+    // in a follow-up cleanup migration.
+    // `raw_hash` is BLAKE3 of the sorted `name=value` pairs of every
+    // header (stable + volatile); `stable_hash` is BLAKE3 of the sorted
+    // `name=value` pairs of stable headers only.
+    // Both `raw_json_hash` and `raw_hash` are UNIQUE so a JS-side upsert
+    // can guarantee dedup — duplicate raw JSON strings map to the same
+    // row and same PK.
+    // `idx_header_sets_stable` accelerates the "how many pages share this
+    // stable header profile" lookup that 0.13 readers use.
+    await instance.raw(`
+		CREATE TABLE header_sets (
+			id                 INTEGER PRIMARY KEY,
+			raw_json_hash      BLOB NOT NULL UNIQUE,
+			raw_hash           BLOB NOT NULL UNIQUE,
+			stable_hash        BLOB NOT NULL,
+			volatile_hash      BLOB,
+			entry_count        INTEGER NOT NULL,
+			stable_entry_count INTEGER NOT NULL
+		)
+	`);
+    await instance.raw('CREATE INDEX idx_header_sets_stable ON header_sets(stable_hash)');
+    // `occurrence` is the 1-based index among entries sharing the same
+    // `(header_set_id, name_id)` pair. HTTP allows a header name to appear
+    // multiple times in one response (e.g. multiple `Set-Cookie` lines);
+    // the composite PK preserves all values without truncation.
+    await instance.raw(`
+		CREATE TABLE header_set_entries (
+			header_set_id INTEGER NOT NULL REFERENCES header_sets(id),
+			name_id       INTEGER NOT NULL REFERENCES header_name_refs(id),
+			occurrence    INTEGER NOT NULL,
+			value_id      INTEGER NOT NULL REFERENCES header_value_refs(id),
+			is_volatile   INTEGER NOT NULL,
+			PRIMARY KEY(header_set_id, name_id, occurrence)
+		) WITHOUT ROWID
+	`);
+    // Pre-computed booleans for the security / caching header checks that
+    // `checkHeaders` currently expresses as per-row LIKE predicates over
+    // `responseHeaders`. Populated in 0.13 by reusing
+    // `headerPresenceExpression`'s logic so the flag rules stay in one
+    // place. `cache_policy` is a short summary string captured for display;
+    // nullable when the header is absent.
+    await instance.raw(`
+		CREATE TABLE header_flags (
+			header_set_id              INTEGER PRIMARY KEY REFERENCES header_sets(id),
+			has_csp                    INTEGER NOT NULL,
+			has_x_frame_options        INTEGER NOT NULL,
+			has_x_content_type_options INTEGER NOT NULL,
+			has_hsts                   INTEGER NOT NULL,
+			has_referrer_policy        INTEGER NOT NULL,
+			has_permissions_policy     INTEGER NOT NULL,
+			has_set_cookie             INTEGER NOT NULL,
+			cache_policy               TEXT
+		)
+	`);
+}