@nitpicker/crawler 0.11.0 → 0.13.0

package/lib/archive/populate-entity-tables/test-utils/setup-entities-db.d.ts

@@ -0,0 +1,23 @@
+import knex from 'knex';
+/**
+ * Provisions an in-memory SQLite database with:
+ *
+ * - A minimal subset of the legacy write model (`pages`, `resources`,
+ *   `anchors`, `images`, `resources-referrers`, `page_html_ref`,
+ *   `page_html_blobs`) — only the columns the 0.13 populates
+ *   actually read.
+ * - The 0.13 ref / header tables (via {@link createRefTables}).
+ * - The 0.13 entity tables (via {@link createEntityTables}).
+ *
+ * Every 0.13 populate spec calls this to obtain a fresh DB. The
+ * caller is responsible for `db.destroy()` (spec `afterEach`).
+ *
+ * `PRAGMA foreign_keys = ON` is enabled at setup time so specs mirror
+ * the migration script's execution mode. `content_items.redirect_dest_id`'s
+ * DEFERRABLE FK is enforced at COMMIT, so specs that exercise the
+ * self-reference must run their inserts inside a `db.transaction`
+ * boundary (see `populate-content-items.spec.ts`'s redirect test) —
+ * bare inserts would trip the FK immediately.
+ * @returns Connected Knex instance.
+ */
+export declare function setupMigrationDb(): Promise<ReturnType<typeof knex>>;

package/lib/archive/populate-entity-tables/test-utils/setup-entities-db.js

@@ -0,0 +1,178 @@
+import knex from 'knex';
+import { createEntityTables } from '../../create-entity-tables.js';
+import { createRefTables } from '../../create-ref-tables.js';
+import { LibsqlDialect } from '../../libsql-dialect.js';
+/**
+ * Provisions an in-memory SQLite database with:
+ *
+ * - A minimal subset of the legacy write model (`pages`, `resources`,
+ *   `anchors`, `images`, `resources-referrers`, `page_html_ref`,
+ *   `page_html_blobs`) — only the columns the 0.13 populates
+ *   actually read.
+ * - The 0.13 ref / header tables (via {@link createRefTables}).
+ * - The 0.13 entity tables (via {@link createEntityTables}).
+ *
+ * Every 0.13 populate spec calls this to obtain a fresh DB. The
+ * caller is responsible for `db.destroy()` (spec `afterEach`).
+ *
+ * `PRAGMA foreign_keys = ON` is enabled at setup time so specs mirror
+ * the migration script's execution mode. `content_items.redirect_dest_id`'s
+ * DEFERRABLE FK is enforced at COMMIT, so specs that exercise the
+ * self-reference must run their inserts inside a `db.transaction`
+ * boundary (see `populate-content-items.spec.ts`'s redirect test) —
+ * bare inserts would trip the FK immediately.
+ * @returns Connected Knex instance.
+ */
+export async function setupMigrationDb() {
+    const db = knex({
+        client: LibsqlDialect,
+        connection: { filename: ':memory:' },
+        useNullAsDefault: true,
+    });
+    await db.raw('PRAGMA foreign_keys = ON');
+    await db.schema.createTable('pages', (t) => {
+        t.increments('id');
+        t.string('url').notNullable();
+        t.integer('redirectDestId').nullable();
+        t.boolean('scraped').notNullable();
+        t.boolean('isTarget').notNullable();
+        t.boolean('isExternal');
+        t.integer('status');
+        t.string('statusText');
+        t.string('contentType').nullable();
+        t.integer('contentLength').nullable();
+        t.json('responseHeaders').nullable();
+        t.string('lang');
+        t.string('dir');
+        t.string('charset');
+        t.string('baseHref');
+        t.text('viewport_raw');
+        t.string('themeColor');
+        t.string('applicationName');
+        t.string('author');
+        t.string('generator');
+        t.string('publisher');
+        t.string('title');
+        t.text('description');
+        t.text('keywords');
+        t.text('robots_raw');
+        t.integer('robots_noindex');
+        t.integer('robots_nofollow');
+        t.integer('robots_noarchive');
+        t.integer('robots_noimageindex');
+        t.string('googlebot');
+        t.string('canonical');
+        t.string('amphtml');
+        t.string('manifest');
+        t.string('icon_href');
+        t.string('appleTouchIcon_href');
+        t.string('og_type');
+        t.string('og_title');
+        t.string('og_url');
+        t.string('og_site_name');
+        t.text('og_description');
+        t.string('og_image');
+        t.string('og_image_alt');
+        t.string('og_image_width');
+        t.string('og_image_height');
+        t.string('og_locale');
+        t.string('og_article_published_time');
+        t.string('og_article_modified_time');
+        t.string('twitter_card');
+        t.string('twitter_site');
+        t.string('twitter_creator');
+        t.string('twitter_title');
+        t.text('twitter_description');
+        t.string('twitter_image');
+        t.string('fb_app_id');
+        t.string('verification_google');
+        t.integer('formatDetection_telephone');
+        t.integer('firstCrawledAt');
+        t.integer('lastCrawledAt');
+        t.integer('tag_count');
+        t.integer('jsonld_count');
+        t.text('tags_providers_csv');
+        t.json('meta_extras');
+        t.boolean('isSkipped');
+        t.string('skipReason');
+        t.integer('order');
+        t.string('source').notNullable().defaultTo('crawled');
+    });
+    await db.schema.createTable('resources', (t) => {
+        t.increments('id');
+        t.string('url').notNullable();
+        t.boolean('isExternal');
+        t.integer('status');
+        t.string('statusText');
+        t.string('contentType').nullable();
+        t.integer('contentLength').nullable();
+        t.string('compress').nullable();
+        t.string('cdn').nullable();
+        t.json('responseHeaders').nullable();
+        t.string('source').notNullable().defaultTo('crawled');
+    });
+    await db.schema.createTable('anchors', (t) => {
+        t.increments('id');
+        t.integer('pageId').notNullable();
+        t.integer('hrefId').notNullable();
+        t.string('hash');
+        t.string('textContent').nullable();
+    });
+    await db.schema.createTable('images', (t) => {
+        t.increments('id');
+        t.integer('pageId').notNullable();
+        t.string('src').nullable();
+        t.string('currentSrc').nullable();
+        t.string('alt');
+        t.float('width');
+        t.float('height');
+        t.integer('naturalWidth');
+        t.integer('naturalHeight');
+        t.boolean('isLazy');
+        t.integer('viewportWidth');
+        t.string('sourceCode');
+    });
+    await db.schema.createTable('resources-referrers', (t) => {
+        t.increments('id');
+        t.integer('resourceId').notNullable();
+        t.integer('pageId').notNullable();
+        t.unique(['resourceId', 'pageId']);
+    });
+    // `analysis_violations` is populated by analyze plugins downstream of
+    // crawl; the migration itself does not create rows here, but
+    // `checkReaderParity` joins it to `pages` / `content_items` so the
+    // table must exist so the parity SELECTs return 0-vs-0 (which the
+    // parity check legitimately skips).
+    await db.schema.createTable('analysis_violations', (t) => {
+        t.increments('id');
+        t.integer('page_id').notNullable();
+        t.string('rule').notNullable();
+        t.string('severity').notNullable();
+        t.text('message');
+    });
+    // `populate-image-items.ts` reads HTML BLOBs via the same trx that
+    // owns the outer populate transaction (see that file's JSDoc for the
+    // deadlock the inline read avoids). Provide the two tables so specs
+    // can seed rows or leave them empty (the reader returns `null` on a
+    // missing row, matching the `getHtmlOfPageById` contract). Raw SQL
+    // so the schema mirrors `init-schema.ts` exactly (`WITHOUT ROWID`,
+    // `size_raw` + `size_stored`, `codec` CHECK).
+    await db.raw(`
+		CREATE TABLE page_html_blobs (
+			hash         BLOB PRIMARY KEY,
+			body         BLOB NOT NULL,
+			codec        TEXT NOT NULL CHECK(codec IN ('zstd', 'none')),
+			size_raw     INTEGER NOT NULL,
+			size_stored  INTEGER NOT NULL
+		) WITHOUT ROWID
+	`);
+    await db.raw(`
+		CREATE TABLE page_html_ref (
+			page_id  INTEGER PRIMARY KEY,
+			hash     BLOB NOT NULL REFERENCES page_html_blobs(hash)
+		) WITHOUT ROWID
+	`);
+    await createRefTables(db);
+    await createEntityTables(db);
+    return db;
+}

package/lib/archive/populate-entity-tables/types.d.ts

@@ -0,0 +1,157 @@
+/**
+ * Domain types for the 0.13 entity/edge population step (issue #193).
+ *
+ * The 0.13 populate helpers translate rows from the legacy write
+ * model (`pages`, `resources`, `anchors`, `images`, `resources-referrers`)
+ * into the normalised entity + edge tables (`content_items`, `page_meta`,
+ * `resource_items`, `anchor_edges`, `resource_ref_edges`, `image_items`)
+ * that 0.13 created. Every ref-id lookup goes through a small set of
+ * shared resolvers whose interfaces live here per the repo-wide
+ * `型は types.ts に集約` rule.
+ */
+/**
+ * Mapping from `url_refs.url` (the natural key) to `url_refs.id`. Populated
+ * on demand from the DB by {@link ./resolve-url-refs.ts}. `null` on values
+ * that are stored in {@link ./resolve-blob-refs.ts} instead (large data
+ * URIs — see the routing rule in
+ * {@link ../populate-ref-tables/populate-url-refs.ts}) or on lookup miss.
+ */
+export interface UrlRefResolution {
+    /** `url_refs.id` when the URL string is present in the dictionary. */
+    readonly urlId: number | null;
+    /** `blob_refs.id` when the value is a large data URI routed to blob_refs. */
+    readonly blobId: number | null;
+}
+/**
+ * Batch-resolved `text_refs.id` for a set of raw text strings. Keyed by
+ * the raw text verbatim so the caller does not need to re-hash for
+ * cache probes.
+ */
+export type TextRefIdMap = ReadonlyMap<string, number>;
+/**
+ * Batch-resolved `url_refs.id` for a set of URL strings. Keyed by the
+ * raw URL verbatim; missing entries mean the URL is either a large data
+ * URI (routed to `blob_refs`) or absent from the dictionary — the caller
+ * distinguishes via a separate `blob_refs` probe.
+ */
+export type UrlRefIdMap = ReadonlyMap<string, number>;
+/**
+ * Batch-resolved `blob_refs.id` for a set of large data URI values.
+ * Keyed by the raw value verbatim (the data URI string), consistent with
+ * how `populateBlobRefs` inserts one row per distinct data URI payload
+ * hash — the resolver looks up by SHA-256 of the decoded payload but
+ * returns the map keyed by the raw URI so callers do not have to keep
+ * the decoded bytes around.
+ */
+export type BlobRefIdMap = ReadonlyMap<string, number>;
+/**
+ * Batch-resolved `content_type_refs.id` for a set of raw content-type
+ * header values. Keyed by the raw value exactly as stored in
+ * `pages.contentType` / `resources.contentType`; missing entries mean
+ * the content type is null or absent from the dictionary (which should
+ * not happen after `populateContentTypeRefs` populates every distinct
+ * value).
+ */
+export type ContentTypeRefIdMap = ReadonlyMap<string, number>;
+/**
+ * Batch-resolved `header_sets.id` for a set of raw `responseHeaders`
+ * JSON strings. Keyed by the raw JSON string; the resolver hashes each
+ * value in JS and looks up `header_sets.raw_json_hash` because SQLite
+ * has no built-in BLAKE3 / SHA-256 (see
+ * {@link ../populate-ref-tables/compute-content-hash.ts} for the algorithm choice).
+ */
+export type HeaderSetIdMap = ReadonlyMap<string, number>;
+/**
+ * One collapsed anchor edge produced by the single-pass keyset scan in
+ * {@link ./populate-anchor-edges.ts}. Rows are emitted in `(page_id,
+ * href_page_id)` order; `first_text_id` is resolved in a second pass
+ * after `text_refs` lookups.
+ */
+export interface AnchorEdgeRow {
+    /** `content_items.id` of the page containing the anchor. */
+    page_id: number;
+    /** `content_items.id` of the anchor target. */
+    href_page_id: number;
+    /** Number of `anchors` rows collapsed into this edge. */
+    count: number;
+    /** `first_hash` — the `anchors.hash` of the first (lowest-id) instance. */
+    first_hash: string | null;
+    /**
+     * `first_text_id` — resolved from `first_textContent` against
+     * `text_refs`. `null` when the first instance had no textContent (an
+     * empty `<a>` tag) or when the text failed to resolve.
+     */
+    first_text_id: number | null;
+}
+/**
+ * One input row for {@link ./collapse-anchor-rows.ts}. Every field
+ * mirrors the legacy `anchors` schema; the collapser does not depend on
+ * knex or the DB so it can be unit-tested with plain arrays.
+ */
+export interface AnchorInputRow {
+    /** Legacy `anchors.id` — used only to sort input; not stored on the edge. */
+    id: number;
+    /** Legacy `anchors.pageId` — `content_items.id` of the source page. */
+    pageId: number;
+    /** Legacy `anchors.hrefId` — `content_items.id` of the anchor target. */
+    hrefId: number;
+    /** Legacy `anchors.hash` — verbatim SHA-256 hex from beholder. */
+    hash: string | null;
+    /** Legacy `anchors.textContent` — anchor visible text (empty for `<a>` with no content). */
+    textContent: string | null;
+}
+/**
+ * Intermediate shape yielded by {@link ./collapse-anchor-rows.ts}. Differs
+ * from the final {@link AnchorEdgeRow} in that `first_text_id` has not
+ * been resolved yet — the raw text is carried through as
+ * `first_textContent` so the second pass can batch-look up every
+ * distinct text against `text_refs`.
+ */
+export interface AnchorEdgeRowInProgress {
+    /** `content_items.id` of the page. */
+    page_id: number;
+    /** `content_items.id` of the anchor target. */
+    href_page_id: number;
+    /** Number of collapsed `anchors` rows. */
+    count: number;
+    /** Verbatim `anchors.hash` of the first (lowest-id) instance. */
+    first_hash: string | null;
+    /**
+     * Verbatim `anchors.textContent` of the first instance — carried
+     * unresolved so the caller can batch-look up ids across every edge.
+     */
+    first_textContent: string | null;
+}
+/**
+ * Case discriminator for {@link ./match-images-to-dom-paths.ts}: which of
+ * the three resolution paths produced the returned `dom_path` string. The
+ * migration script's jsdom resolver emits a diagnostic warning log for
+ * `unknown` fallbacks so operators can audit reconstruction fidelity.
+ */
+export type DomPathDerivationCase = 'single-match' | 'ordinal-match' | 'unknown';
+/**
+ * Result of the DOM-path derivation for one legacy `images` row.
+ * `path` is the string to insert into `text_refs.text`; `case` records
+ * which branch produced it.
+ */
+export interface DomPathResult {
+    /** The `dom_path` string, e.g. `html/body[1]/main[1]/img[3]`. */
+    path: string;
+    /** Which branch produced the string. */
+    case: DomPathDerivationCase;
+}
+/**
+ * One `<img>` observed in a rendered / parsed document, in document
+ * order: its `outerHTML` (the matching key against the crawler's stored
+ * `sourceCode`) and its pre-derived `dom_path` string. Produced either
+ * by a jsdom walk over an archived HTML snapshot (migration script) or
+ * by an in-browser walk over the live page (crawler write path) — both
+ * feed {@link ./match-images-to-dom-paths.ts}, which is DOM-free and
+ * only compares these strings.
+ */
+export interface DomPathCandidate {
+    /** The element's `outerHTML`, verbatim. */
+    outerHTML: string;
+    /** The element's derived `dom_path` (see {@link ./derive-dom-path.ts}). */
+    path: string;
+}

package/lib/archive/populate-entity-tables/types.js

@@ -0,0 +1,12 @@
+/**
+ * Domain types for the 0.13 entity/edge population step (issue #193).
+ *
+ * The 0.13 populate helpers translate rows from the legacy write
+ * model (`pages`, `resources`, `anchors`, `images`, `resources-referrers`)
+ * into the normalised entity + edge tables (`content_items`, `page_meta`,
+ * `resource_items`, `anchor_edges`, `resource_ref_edges`, `image_items`)
+ * that 0.13 created. Every ref-id lookup goes through a small set of
+ * shared resolvers whose interfaces live here per the repo-wide
+ * `型は types.ts に集約` rule.
+ */
+export {};

package/lib/archive/populate-entity-tables/upsert-text-refs.d.ts

@@ -0,0 +1,38 @@
+import type { TextRefIdMap } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Upserts a set of text strings into `text_refs` and returns their ids
+ * (issue #193).
+ *
+ * Unlike {@link ./resolve-text-refs.ts}, which only reads, this helper
+ * inserts every missing string first. It exists specifically for
+ * `dom_path` strings, which are synthesised by the image-items populate
+ * ({@link ./populate-image-items.ts}) and are not among the columns the
+ * `text_refs` populate scans
+ * ({@link ../populate-ref-tables/populate-text-refs.ts}) — the caller
+ * cannot rely on the dictionary already containing them.
+ *
+ * Strategy:
+ *
+ * 1. **Hash and INSERT OR IGNORE** every input string in bulk. New rows
+ *    materialise; conflicts are silently skipped.
+ * 2. **Read back** by hash + text (the `(hash, text)` UNIQUE composite)
+ *    to pick up both freshly-inserted and pre-existing ids.
+ *
+ * The read-back narrows on both hash and text (same defensive shape as
+ * {@link ../populate-ref-tables/populate-header-tables.ts}'s `resolveValueIds`) so
+ * an astronomically improbable hash collision cannot leak the wrong
+ * id back to the caller.
+ *
+ * Empty / null strings are ignored; duplicates in `texts` are deduped
+ * internally.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param texts - Iterable of text strings to upsert.
+ * @returns Map keyed by the raw text string.
+ * @example
+ * const idMap = await upsertTextRefs(trx, [
+ *   'html/body[1]/img[1]',
+ *   'unknown/42',
+ * ]);
+ */
+export declare function upsertTextRefs(trx: Knex, texts: Iterable<string>): Promise<TextRefIdMap>;

package/lib/archive/populate-entity-tables/upsert-text-refs.js

@@ -0,0 +1,78 @@
+import { computeContentHash } from '../populate-ref-tables/compute-content-hash.js';
+/**
+ * Rows sent per `INSERT INTO text_refs ... VALUES (...)` statement. Each
+ * row binds 2 params (hash + text) so 500 rows = 1 000 params — well
+ * under the SQLite variable limit.
+ */
+const INSERT_CHUNK_SIZE = 500;
+/**
+ * Rows sent per `SELECT ... WHERE hash IN (?, ...)` chunk on the
+ * post-insert read-back. Same reasoning as {@link ./resolve-text-refs.ts}.
+ */
+const LOOKUP_CHUNK_SIZE = 800;
+/**
+ * Upserts a set of text strings into `text_refs` and returns their ids
+ * (issue #193).
+ *
+ * Unlike {@link ./resolve-text-refs.ts}, which only reads, this helper
+ * inserts every missing string first. It exists specifically for
+ * `dom_path` strings, which are synthesised by the image-items populate
+ * ({@link ./populate-image-items.ts}) and are not among the columns the
+ * `text_refs` populate scans
+ * ({@link ../populate-ref-tables/populate-text-refs.ts}) — the caller
+ * cannot rely on the dictionary already containing them.
+ *
+ * Strategy:
+ *
+ * 1. **Hash and INSERT OR IGNORE** every input string in bulk. New rows
+ *    materialise; conflicts are silently skipped.
+ * 2. **Read back** by hash + text (the `(hash, text)` UNIQUE composite)
+ *    to pick up both freshly-inserted and pre-existing ids.
+ *
+ * The read-back narrows on both hash and text (same defensive shape as
+ * {@link ../populate-ref-tables/populate-header-tables.ts}'s `resolveValueIds`) so
+ * an astronomically improbable hash collision cannot leak the wrong
+ * id back to the caller.
+ *
+ * Empty / null strings are ignored; duplicates in `texts` are deduped
+ * internally.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param texts - Iterable of text strings to upsert.
+ * @returns Map keyed by the raw text string.
+ * @example
+ * const idMap = await upsertTextRefs(trx, [
+ *   'html/body[1]/img[1]',
+ *   'unknown/42',
+ * ]);
+ */
+export async function upsertTextRefs(trx, texts) {
+    const distinct = new Set();
+    for (const text of texts) {
+        if (typeof text === 'string' && text !== '') {
+            distinct.add(text);
+        }
+    }
+    if (distinct.size === 0) {
+        return new Map();
+    }
+    const values = [...distinct];
+    const hashes = values.map((text) => computeContentHash(text));
+    const inserts = values.map((text, index) => ({ hash: hashes[index], text }));
+    for (let index = 0; index < inserts.length; index += INSERT_CHUNK_SIZE) {
+        const chunk = inserts.slice(index, index + INSERT_CHUNK_SIZE);
+        await trx('text_refs').insert(chunk).onConflict(['hash', 'text']).ignore();
+    }
+    const result = new Map();
+    for (let index = 0; index < values.length; index += LOOKUP_CHUNK_SIZE) {
+        const chunkValues = values.slice(index, index + LOOKUP_CHUNK_SIZE);
+        const chunkHashes = hashes.slice(index, index + LOOKUP_CHUNK_SIZE);
+        const rows = await trx('text_refs')
+            .select('id', 'hash', 'text')
+            .whereIn('hash', chunkHashes)
+            .whereIn('text', chunkValues);
+        for (const row of rows) {
+            result.set(row.text, row.id);
+        }
+    }
+    return result;
+}

package/lib/archive/populate-ref-tables/classify-content-type.d.ts

@@ -0,0 +1,16 @@
+import type { ContentTypeCategory } from './types.js';
+/**
+ * Classifies a raw HTTP `Content-Type` header value into the same coarse
+ * categories used by the viewer read-model. The rule table in
+ * `content-type-rules.ts` is evaluated in declared order and the first
+ * matching rule wins (so `application/xhtml+xml` → `html` and
+ * `image/svg+xml` → `image`).
+ *
+ * `null` / empty / whitespace-only inputs map to `'unknown'`; MIMEs that
+ * fail every rule map to `'other'`. Behaviour mirrors
+ * `@nitpicker/query`'s `classifyContentType`; see the type header of
+ * `content-type-rules.ts` for the drift-warning contract.
+ * @param contentType - The raw `Content-Type` header value, or `null`.
+ * @returns The canonical category label.
+ */
+export declare function classifyContentType(contentType: string | null): ContentTypeCategory;

package/lib/archive/populate-ref-tables/classify-content-type.js

@@ -0,0 +1,52 @@
+import { CONTENT_TYPE_RULES } from './content-type-rules.js';
+/**
+ * Classifies a raw HTTP `Content-Type` header value into the same coarse
+ * categories used by the viewer read-model. The rule table in
+ * `content-type-rules.ts` is evaluated in declared order and the first
+ * matching rule wins (so `application/xhtml+xml` → `html` and
+ * `image/svg+xml` → `image`).
+ *
+ * `null` / empty / whitespace-only inputs map to `'unknown'`; MIMEs that
+ * fail every rule map to `'other'`. Behaviour mirrors
+ * `@nitpicker/query`'s `classifyContentType`; see the type header of
+ * `content-type-rules.ts` for the drift-warning contract.
+ * @param contentType - The raw `Content-Type` header value, or `null`.
+ * @returns The canonical category label.
+ */
+export function classifyContentType(contentType) {
+    if (contentType == null) {
+        return 'unknown';
+    }
+    const semi = contentType.indexOf(';');
+    const head = (semi === -1 ? contentType : contentType.slice(0, semi))
+        .trim()
+        .toLowerCase();
+    if (head === '') {
+        return 'unknown';
+    }
+    for (const rule of CONTENT_TYPE_RULES) {
+        for (const matcher of rule.matchers) {
+            switch (matcher.kind) {
+                case 'exact': {
+                    if (head === matcher.value) {
+                        return rule.category;
+                    }
+                    break;
+                }
+                case 'prefix': {
+                    if (head.startsWith(matcher.value)) {
+                        return rule.category;
+                    }
+                    break;
+                }
+                case 'suffix': {
+                    if (head.endsWith(matcher.value)) {
+                        return rule.category;
+                    }
+                    break;
+                }
+            }
+        }
+    }
+    return 'other';
+}

package/lib/archive/populate-ref-tables/compute-content-hash.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Computes a 32-byte content hash for a UTF-8 string or raw byte buffer.
+ *
+ * The hash columns on `text_refs`, `json_refs`, `blob_refs`,
+ * `header_value_refs`, and `header_sets` use **SHA-256** rather than a
+ * faster candidate such as BLAKE3, for three reasons:
+ *
+ * 1. **No new dependency**. `@nitpicker/crawler`'s dependency list is small
+ *    and CLAUDE.md's supply-chain rules discourage adding a hash-only lib.
+ * 2. **Consistency**. `page_html_blobs.hash` — the existing content-addressable
+ *    dictionary — is SHA-256 (see `Database.#writePageHtmlBlob`). Using the
+ *    same algorithm keeps every content-hash column in the archive on one
+ *    scheme.
+ * 3. **Column contract is length-only**. Every 0.13 hash column is typed
+ *    `BLOB` (32 bytes when populated). SHA-256 satisfies that contract.
+ *
+ * If BLAKE3 is later required for throughput, this single function is the only
+ * edit point — every callsite in 0.13 routes through here.
+ * @param value - Value to hash. `string` inputs are encoded as UTF-8.
+ * @returns 32-byte hash as a `Buffer`, ready to insert into a `BLOB` column.
+ */
+export declare function computeContentHash(value: string | Uint8Array): Buffer;

package/lib/archive/populate-ref-tables/compute-content-hash.js

@@ -0,0 +1,26 @@
+import { createHash } from 'node:crypto';
+/**
+ * Computes a 32-byte content hash for a UTF-8 string or raw byte buffer.
+ *
+ * The hash columns on `text_refs`, `json_refs`, `blob_refs`,
+ * `header_value_refs`, and `header_sets` use **SHA-256** rather than a
+ * faster candidate such as BLAKE3, for three reasons:
+ *
+ * 1. **No new dependency**. `@nitpicker/crawler`'s dependency list is small
+ *    and CLAUDE.md's supply-chain rules discourage adding a hash-only lib.
+ * 2. **Consistency**. `page_html_blobs.hash` — the existing content-addressable
+ *    dictionary — is SHA-256 (see `Database.#writePageHtmlBlob`). Using the
+ *    same algorithm keeps every content-hash column in the archive on one
+ *    scheme.
+ * 3. **Column contract is length-only**. Every 0.13 hash column is typed
+ *    `BLOB` (32 bytes when populated). SHA-256 satisfies that contract.
+ *
+ * If BLAKE3 is later required for throughput, this single function is the only
+ * edit point — every callsite in 0.13 routes through here.
+ * @param value - Value to hash. `string` inputs are encoded as UTF-8.
+ * @returns 32-byte hash as a `Buffer`, ready to insert into a `BLOB` column.
+ */
+export function computeContentHash(value) {
+    const bytes = typeof value === 'string' ? Buffer.from(value, 'utf8') : value;
+    return createHash('sha256').update(bytes).digest();
+}

package/lib/archive/populate-ref-tables/compute-header-flags.d.ts

@@ -0,0 +1,16 @@
+import type { HeaderEntry, HeaderFlagsRow } from './types.js';
+/**
+ * Computes the {@link HeaderFlagsRow} for one decomposed header set.
+ *
+ * Each `has_*` boolean is a straight "is this name present" check on the
+ * decomposed entries — semantically equivalent to `checkHeaders` /
+ * `headerPresenceExpression` in `@nitpicker/query`, but expressed in JS
+ * over the already-normalized `HeaderEntry.name` (lower-cased) so no
+ * `LIKE` pattern is needed. This drops the JSON-key-shape false-positive
+ * risk that `headerPresenceExpression`'s SQL guard against (a value that
+ * happens to mention another header's name won't fire here because names
+ * come from the parsed JSON's KEYS, not from a substring scan).
+ * @param entries - Every decomposed entry for one `header_sets` row.
+ * @returns The row-shaped flags to insert into `header_flags`.
+ */
+export declare function computeHeaderFlags(entries: readonly HeaderEntry[]): HeaderFlagsRow;