@nitpicker/crawler 0.11.0 → 0.13.0

package/lib/archive/db-ops/_shared/upsert-blob-ref.js

@@ -0,0 +1,48 @@
+import { zstdCompressSync } from 'node:zlib';
+import { computeContentHash } from '../../populate-ref-tables/compute-content-hash.js';
+import { decodeDataUri } from '../../populate-ref-tables/decode-data-uri.js';
+/**
+ * Resolves the `blob_refs.id` for one large `data:` URI, decoding and
+ * inserting the deduplicated (hash-keyed) payload row when it is new.
+ *
+ * Callers route values here per the data-URI threshold rule (`data:` URIs
+ * longer than `DATA_URI_URL_REFS_LIMIT` — everything else belongs in
+ * `url_refs`). The payload bytes are decoded from the base64 /
+ * percent-encoded tail, hashed, and stored zstd-compressed — the same
+ * storage contract as the rows produced when migrating an existing
+ * archive (`populate-ref-tables/populate-blob-refs.ts`).
+ *
+ * Returns `null` when the data URI fails to decode (malformed base64 /
+ * truncated percent escape). A single malformed image `src` must not
+ * abort the page write; the caller stores `*_blob_id = null` and the
+ * information loss is limited to that one slot.
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ * @param caches - The connection's write-side id caches; mutated in place.
+ * @param dataUri - The raw `data:` URI value.
+ * @returns The `blob_refs.id`, or `null` when the URI cannot be decoded.
+ * @example
+ * const id = await upsertBlobRef(knex, caches, 'data:image/png;base64,<payload>');
+ */
+export async function upsertBlobRef(qb, caches, dataUri) {
+    const decoded = decodeDataUri(dataUri);
+    if (decoded === null) {
+        return null;
+    }
+    const hash = computeContentHash(decoded.bytes);
+    const hex = hash.toString('hex');
+    const cached = caches.blobIds.get(hex);
+    if (cached !== undefined) {
+        return cached;
+    }
+    const compressed = zstdCompressSync(decoded.bytes);
+    const rows = await qb.raw(`INSERT INTO blob_refs (hash, body, codec, size_raw, size_stored)
+		 VALUES (?, ?, 'zstd', ?, ?)
+		 ON CONFLICT(hash) DO UPDATE SET hash = hash
+		 RETURNING id`, [hash, compressed, decoded.bytes.byteLength, compressed.byteLength]);
+    const first = rows[0];
+    if (first === undefined) {
+        throw new Error('upsertBlobRef: RETURNING yielded no row');
+    }
+    caches.blobIds.set(hex, first.id);
+    return first.id;
+}

package/lib/archive/db-ops/_shared/upsert-content-type-ref.d.ts

@@ -0,0 +1,30 @@
+import type { WriteRefCaches } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Resolves the `content_type_refs.id` for one content-type value,
+ * inserting the dictionary row when the value is new.
+ *
+ * `raw` is expected to be the already-canonicalised form produced by
+ * `normalizeContentType` (lower-cased, trimmed) — the write path
+ * canonicalises before storing, exactly as the legacy `pages.contentType`
+ * column did, so the dictionary's natural key never forks on case or
+ * whitespace variants of the same MIME. `normalized` is derived via the
+ * shared {@link ../../populate-ref-tables/normalize-mime.ts} (the same
+ * derivation the archive-migration populate uses, so live-crawl rows and
+ * migrated rows never fork on the same raw value); `category` via
+ * {@link ../../populate-ref-tables/classify-content-type.ts}.
+ *
+ * The content-type space is tiny (well under 1 000 distinct values on
+ * even the largest archives) so the cache converges after the first few
+ * pages and steady-state calls issue no SQL.
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ * @param caches - The connection's write-side id caches; mutated in place.
+ * @param raw - Canonicalised content-type value. Pass the output of
+ *   `normalizeContentType`; `null` / empty input must be short-circuited
+ *   by the caller (a row without a content type stores `content_type_id
+ *   = null`, not a dictionary entry).
+ * @returns The `content_type_refs.id` of the existing or new row.
+ * @example
+ * const id = await upsertContentTypeRef(knex, caches, 'text/html');
+ */
+export declare function upsertContentTypeRef(qb: Knex | Knex.Transaction, caches: WriteRefCaches, raw: string): Promise<number>;

package/lib/archive/db-ops/_shared/upsert-content-type-ref.js

@@ -0,0 +1,45 @@
+import { classifyContentType } from '../../populate-ref-tables/classify-content-type.js';
+import { normalizeMime } from '../../populate-ref-tables/normalize-mime.js';
+/**
+ * Resolves the `content_type_refs.id` for one content-type value,
+ * inserting the dictionary row when the value is new.
+ *
+ * `raw` is expected to be the already-canonicalised form produced by
+ * `normalizeContentType` (lower-cased, trimmed) — the write path
+ * canonicalises before storing, exactly as the legacy `pages.contentType`
+ * column did, so the dictionary's natural key never forks on case or
+ * whitespace variants of the same MIME. `normalized` is derived via the
+ * shared {@link ../../populate-ref-tables/normalize-mime.ts} (the same
+ * derivation the archive-migration populate uses, so live-crawl rows and
+ * migrated rows never fork on the same raw value); `category` via
+ * {@link ../../populate-ref-tables/classify-content-type.ts}.
+ *
+ * The content-type space is tiny (well under 1 000 distinct values on
+ * even the largest archives) so the cache converges after the first few
+ * pages and steady-state calls issue no SQL.
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ * @param caches - The connection's write-side id caches; mutated in place.
+ * @param raw - Canonicalised content-type value. Pass the output of
+ *   `normalizeContentType`; `null` / empty input must be short-circuited
+ *   by the caller (a row without a content type stores `content_type_id
+ *   = null`, not a dictionary entry).
+ * @returns The `content_type_refs.id` of the existing or new row.
+ * @example
+ * const id = await upsertContentTypeRef(knex, caches, 'text/html');
+ */
+export async function upsertContentTypeRef(qb, caches, raw) {
+    const cached = caches.contentTypeIds.get(raw);
+    if (cached !== undefined) {
+        return cached;
+    }
+    const rows = await qb.raw(`INSERT INTO content_type_refs (raw, normalized, category)
+		 VALUES (?, ?, ?)
+		 ON CONFLICT(raw) DO UPDATE SET raw = raw
+		 RETURNING id`, [raw, normalizeMime(raw), classifyContentType(raw)]);
+    const first = rows[0];
+    if (first === undefined) {
+        throw new Error(`upsertContentTypeRef: RETURNING yielded no row for ${raw}`);
+    }
+    caches.contentTypeIds.set(raw, first.id);
+    return first.id;
+}

package/lib/archive/db-ops/_shared/upsert-json-ref.d.ts

@@ -0,0 +1,22 @@
+import type { WriteRefCaches } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Resolves the `json_refs.id` for one JSON payload string, inserting the
+ * deduplicated (hash-keyed) row when the payload is new.
+ *
+ * The payload is hashed over its raw UTF-8 bytes and stored
+ * zstd-compressed (`codec = 'zstd'`) with `size_raw` / `size_stored`
+ * bookkeeping — the same storage contract as `page_html_blobs` and the
+ * dictionary rows produced when migrating an existing archive
+ * (`populate-ref-tables/populate-json-refs.ts`). Identical payloads
+ * (e.g. the same `meta_extras` JSON across templated pages) produce one
+ * row; the compression cost is paid only on a cache-and-DB miss.
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ * @param caches - The connection's write-side id caches; mutated in place.
+ * @param jsonText - Raw JSON string. Empty / null input must be
+ *   short-circuited by the caller (`*_json_id = null`).
+ * @returns The `json_refs.id` of the existing or new row.
+ * @example
+ * const id = await upsertJsonRef(knex, caches, JSON.stringify(extras));
+ */
+export declare function upsertJsonRef(qb: Knex | Knex.Transaction, caches: WriteRefCaches, jsonText: string): Promise<number>;

package/lib/archive/db-ops/_shared/upsert-json-ref.js

@@ -0,0 +1,41 @@
+import { zstdCompressSync } from 'node:zlib';
+import { computeContentHash } from '../../populate-ref-tables/compute-content-hash.js';
+/**
+ * Resolves the `json_refs.id` for one JSON payload string, inserting the
+ * deduplicated (hash-keyed) row when the payload is new.
+ *
+ * The payload is hashed over its raw UTF-8 bytes and stored
+ * zstd-compressed (`codec = 'zstd'`) with `size_raw` / `size_stored`
+ * bookkeeping — the same storage contract as `page_html_blobs` and the
+ * dictionary rows produced when migrating an existing archive
+ * (`populate-ref-tables/populate-json-refs.ts`). Identical payloads
+ * (e.g. the same `meta_extras` JSON across templated pages) produce one
+ * row; the compression cost is paid only on a cache-and-DB miss.
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ * @param caches - The connection's write-side id caches; mutated in place.
+ * @param jsonText - Raw JSON string. Empty / null input must be
+ *   short-circuited by the caller (`*_json_id = null`).
+ * @returns The `json_refs.id` of the existing or new row.
+ * @example
+ * const id = await upsertJsonRef(knex, caches, JSON.stringify(extras));
+ */
+export async function upsertJsonRef(qb, caches, jsonText) {
+    const rawBytes = Buffer.from(jsonText, 'utf8');
+    const hash = computeContentHash(rawBytes);
+    const hex = hash.toString('hex');
+    const cached = caches.jsonIds.get(hex);
+    if (cached !== undefined) {
+        return cached;
+    }
+    const compressed = zstdCompressSync(rawBytes);
+    const rows = await qb.raw(`INSERT INTO json_refs (hash, json_text, codec, size_raw, size_stored)
+		 VALUES (?, ?, 'zstd', ?, ?)
+		 ON CONFLICT(hash) DO UPDATE SET hash = hash
+		 RETURNING id`, [hash, compressed, rawBytes.byteLength, compressed.byteLength]);
+    const first = rows[0];
+    if (first === undefined) {
+        throw new Error('upsertJsonRef: RETURNING yielded no row');
+    }
+    caches.jsonIds.set(hex, first.id);
+    return first.id;
+}

package/lib/archive/db-ops/_shared/upsert-response-headers.d.ts

@@ -0,0 +1,35 @@
+import type { WriteRefCaches } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Decomposes one response's headers and upserts them into the header
+ * dictionary tables (`header_name_refs` / `header_value_refs` /
+ * `header_sets` / `header_set_entries` / `header_flags`), returning the
+ * `header_sets.id` to store on the referring `content_items` /
+ * `resource_items` row.
+ *
+ * Runs per response (incrementally during the crawl), not deferred to
+ * crawl-end — the header dictionaries grow as pages arrive and the
+ * entity row can reference its `header_set_id` in the same write.
+ * Dedup makes the steady-state cost small: CDN-served sites reuse a
+ * handful of header shapes, so after the first few pages most calls
+ * resolve entirely from the in-process caches (warmed lazily from the DB
+ * on the first call, so a crawl that never sees headers pays nothing).
+ *
+ * Returns `null` for responses with no headers (`null` / `{}` /
+ * unparsable) — the referring row stores `header_set_id = null`, which
+ * is exactly how such rows read back through `header_flags` joins
+ * (absent flags row = no headers recorded).
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ * @param caches - The connection's write-side id caches; the header
+ *   sub-caches are created and attached on first use.
+ * @param responseHeaders - The response's header record as delivered by
+ *   the scraper (`Record<name, string | string[]>`), or `null` /
+ *   `undefined` when the response carried none.
+ * @returns The `header_sets.id`, or `null` when there is no set to store.
+ * @example
+ * const headerSetId = await upsertResponseHeaders(knex, caches, {
+ *   'content-type': 'text/html',
+ *   'cache-control': 'no-store',
+ * });
+ */
+export declare function upsertResponseHeaders(qb: Knex | Knex.Transaction, caches: WriteRefCaches, responseHeaders: Record<string, string | string[] | undefined> | null | undefined): Promise<number | null>;

package/lib/archive/db-ops/_shared/upsert-response-headers.js

@@ -0,0 +1,49 @@
+import { createHeaderTableCaches } from '../../populate-ref-tables/create-header-table-caches.js';
+import { decomposeHeaderSet } from '../../populate-ref-tables/decompose-header-set.js';
+import { upsertOneHeaderSet } from '../../populate-ref-tables/upsert-one-header-set.js';
+/**
+ * Decomposes one response's headers and upserts them into the header
+ * dictionary tables (`header_name_refs` / `header_value_refs` /
+ * `header_sets` / `header_set_entries` / `header_flags`), returning the
+ * `header_sets.id` to store on the referring `content_items` /
+ * `resource_items` row.
+ *
+ * Runs per response (incrementally during the crawl), not deferred to
+ * crawl-end — the header dictionaries grow as pages arrive and the
+ * entity row can reference its `header_set_id` in the same write.
+ * Dedup makes the steady-state cost small: CDN-served sites reuse a
+ * handful of header shapes, so after the first few pages most calls
+ * resolve entirely from the in-process caches (warmed lazily from the DB
+ * on the first call, so a crawl that never sees headers pays nothing).
+ *
+ * Returns `null` for responses with no headers (`null` / `{}` /
+ * unparsable) — the referring row stores `header_set_id = null`, which
+ * is exactly how such rows read back through `header_flags` joins
+ * (absent flags row = no headers recorded).
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ * @param caches - The connection's write-side id caches; the header
+ *   sub-caches are created and attached on first use.
+ * @param responseHeaders - The response's header record as delivered by
+ *   the scraper (`Record<name, string | string[]>`), or `null` /
+ *   `undefined` when the response carried none.
+ * @returns The `header_sets.id`, or `null` when there is no set to store.
+ * @example
+ * const headerSetId = await upsertResponseHeaders(knex, caches, {
+ *   'content-type': 'text/html',
+ *   'cache-control': 'no-store',
+ * });
+ */
+export async function upsertResponseHeaders(qb, caches, responseHeaders) {
+    // Serialise exactly like the scraper's own JSON storage form so the
+    // decomposition (and its raw_json_hash dedup key) matches what a bulk
+    // archive populate would compute for the same response.
+    const rawJson = responseHeaders == null ? null : JSON.stringify(responseHeaders);
+    const decomposed = decomposeHeaderSet(rawJson);
+    if (decomposed === null) {
+        return null;
+    }
+    if (caches.headers === null) {
+        caches.headers = await createHeaderTableCaches(qb);
+    }
+    return await upsertOneHeaderSet(qb, decomposed, caches.headers);
+}

package/lib/archive/db-ops/_shared/upsert-url-ref.d.ts

@@ -0,0 +1,39 @@
+import type { WriteRefCaches } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Resolves the `url_refs.id` for one URL string, inserting the row (with
+ * its decomposed `scheme` / `host` / `port` / `path` / `query_hash` /
+ * `fragment` columns) when the URL is not yet in the dictionary.
+ *
+ * The write path calls this for every URL it touches — page identities,
+ * anchor hrefs, image `src` / `currentSrc`, resource URLs, and URL-shaped
+ * meta columns — so the result is cached in `caches.urlIds` and a cache
+ * hit issues no SQL at all. Anchor hrefs and CDN-hosted image URLs repeat
+ * across thousands of pages, so the hit rate dominates after the first
+ * few pages.
+ *
+ * The cache-miss upsert is a single statement:
+ *
+ * ```sql
+ * INSERT INTO url_refs (...) VALUES (...)
+ * ON CONFLICT(url) DO UPDATE SET url = url
+ * RETURNING id
+ * ```
+ *
+ * `DO UPDATE SET url = url` is a deliberate no-op update whose only
+ * purpose is to make `RETURNING id` yield the existing row's id on
+ * conflict. `INSERT OR IGNORE ... RETURNING id` must NOT be used here:
+ * `RETURNING` returns zero rows when the insert is ignored, which would
+ * leave the caller with no id and a `null` FK on the next entity INSERT.
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ *   Callers inside a transaction MUST pass the `trx` so the read sees
+ *   uncommitted rows from the same transaction.
+ * @param caches - The connection's write-side id caches; mutated in place.
+ * @param url - The URL string to resolve. Stored verbatim as the natural key.
+ * @returns The `url_refs.id` of the existing or newly inserted row.
+ * @example
+ * const id = await upsertUrlRef(knex, caches, 'https://example.com/a');
+ * const again = await upsertUrlRef(knex, caches, 'https://example.com/a');
+ * // id === again, second call hits the cache and issues no SQL
+ */
+export declare function upsertUrlRef(qb: Knex | Knex.Transaction, caches: WriteRefCaches, url: string): Promise<number>;

package/lib/archive/db-ops/_shared/upsert-url-ref.js

@@ -0,0 +1,62 @@
+import { decomposeUrl } from '../../populate-ref-tables/decompose-url.js';
+/**
+ * Resolves the `url_refs.id` for one URL string, inserting the row (with
+ * its decomposed `scheme` / `host` / `port` / `path` / `query_hash` /
+ * `fragment` columns) when the URL is not yet in the dictionary.
+ *
+ * The write path calls this for every URL it touches — page identities,
+ * anchor hrefs, image `src` / `currentSrc`, resource URLs, and URL-shaped
+ * meta columns — so the result is cached in `caches.urlIds` and a cache
+ * hit issues no SQL at all. Anchor hrefs and CDN-hosted image URLs repeat
+ * across thousands of pages, so the hit rate dominates after the first
+ * few pages.
+ *
+ * The cache-miss upsert is a single statement:
+ *
+ * ```sql
+ * INSERT INTO url_refs (...) VALUES (...)
+ * ON CONFLICT(url) DO UPDATE SET url = url
+ * RETURNING id
+ * ```
+ *
+ * `DO UPDATE SET url = url` is a deliberate no-op update whose only
+ * purpose is to make `RETURNING id` yield the existing row's id on
+ * conflict. `INSERT OR IGNORE ... RETURNING id` must NOT be used here:
+ * `RETURNING` returns zero rows when the insert is ignored, which would
+ * leave the caller with no id and a `null` FK on the next entity INSERT.
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ *   Callers inside a transaction MUST pass the `trx` so the read sees
+ *   uncommitted rows from the same transaction.
+ * @param caches - The connection's write-side id caches; mutated in place.
+ * @param url - The URL string to resolve. Stored verbatim as the natural key.
+ * @returns The `url_refs.id` of the existing or newly inserted row.
+ * @example
+ * const id = await upsertUrlRef(knex, caches, 'https://example.com/a');
+ * const again = await upsertUrlRef(knex, caches, 'https://example.com/a');
+ * // id === again, second call hits the cache and issues no SQL
+ */
+export async function upsertUrlRef(qb, caches, url) {
+    const cached = caches.urlIds.get(url);
+    if (cached !== undefined) {
+        return cached;
+    }
+    const decomposed = decomposeUrl(url);
+    const rows = await qb.raw(`INSERT INTO url_refs (url, scheme, host, port, path, query_hash, fragment)
+		 VALUES (?, ?, ?, ?, ?, ?, ?)
+		 ON CONFLICT(url) DO UPDATE SET url = url
+		 RETURNING id`, [
+        url,
+        decomposed.scheme,
+        decomposed.host,
+        decomposed.port,
+        decomposed.path,
+        decomposed.query_hash,
+        decomposed.fragment,
+    ]);
+    const first = rows[0];
+    if (first === undefined) {
+        throw new Error(`upsertUrlRef: RETURNING yielded no row for ${url}`);
+    }
+    caches.urlIds.set(url, first.id);
+    return first.id;
+}

package/lib/archive/db-ops/analysis/replace-analysis-violations.d.ts

@@ -0,0 +1,28 @@
+import type { Knex } from 'knex';
+/**
+ * Replaces the stored analysis violations with a freshly generated set.
+ *
+ * The function resolves every violation URL to a `content_items.id` (via
+ * the `url_refs` dictionary), deduplicates message/code text through
+ * `analysis_text_refs`, and rewrites `analysis_violations` in one
+ * transaction. This is the storage-side counterpart of the query-layer
+ * `getViolations` read path.
+ *
+ * Violations whose `url` still carries the pre-#225 corrupted position
+ * suffix (and that do not already carry explicit `line`/`col`) are repaired
+ * in place: {@link parseLegacyCorruptedUrl} splits the suffix off before URL
+ * resolution, so both freshly-analyzed and legacy-JSON-backfilled data end
+ * up in the same clean shape.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param violations - Flat violation list from the analyze phase.
+ */
+export declare function replaceAnalysisViolations(knex: Knex, violations: readonly {
+    validator: string;
+    severity: string;
+    rule: string;
+    code?: string | null;
+    message: string;
+    url: string;
+    line?: number | null;
+    col?: number | null;
+}[]): Promise<void>;

package/lib/archive/db-ops/analysis/replace-analysis-violations.js

@@ -0,0 +1,152 @@
+import { createHash } from 'node:crypto';
+import { eachSplitted } from '../../../utils/array/each-splitted.js';
+/**
+ * Matches the position suffix that `@nitpicker/analyze-markuplint` and
+ * `@nitpicker/analyze-textlint` used to append to `Violation.url` before
+ * issue #225 (e.g. `https://example.com/page (5:10)`). Anchored to the end
+ * of the string so a URL that legitimately contains a similar-looking
+ * substring earlier on is not mistaken for the suffix.
+ */
+const LEGACY_CORRUPTED_URL_PATTERN = /^(.+) \((\d+):(\d+)\)$/;
+/**
+ * Splits a legacy corrupted `Violation.url` (page URL + trailing
+ * `" (line:col)"`) back into a clean URL and its position, or returns
+ * `null` when `url` does not match the corrupted shape (e.g. a clean axe
+ * URL, or a URL that already carries explicit `line`/`col`).
+ * @param url - The `Violation.url` value as read from storage or input.
+ * @returns The recovered `{ url, line, col }`, or `null` when `url` is not corrupted.
+ */
+function parseLegacyCorruptedUrl(url) {
+    const match = LEGACY_CORRUPTED_URL_PATTERN.exec(url);
+    if (!match) {
+        return null;
+    }
+    const [, cleanUrl, line, col] = match;
+    return { url: cleanUrl, line: Number(line), col: Number(col) };
+}
+/**
+ * Adds the `line`/`col` columns to `analysis_violations` when they are
+ * missing, i.e. when the archive's table was provisioned before issue #225.
+ *
+ * `create-adjunct-tables.ts` never mutates a table it finds already
+ * present — catching up an *existing* table normally requires a
+ * version-gated migration script (see `scripts/migrate-to-0.13.mjs`). This
+ * function relies on the exception documented in ARCHITECTURE.md's
+ * invariants list: a nullable, additive column on a table with a single
+ * write path may self-heal here without a version bump, because
+ * `replaceAnalysisViolations` is that single write path for
+ * `analysis_violations`. Runs before the transaction below so the DDL is
+ * not mixed with the DML rewrite.
+ * @param knex - Knex query builder connected to the archive DB.
+ */
+async function ensureLineColColumns(knex) {
+    if (!(await knex.schema.hasColumn('analysis_violations', 'line'))) {
+        await knex.schema.alterTable('analysis_violations', (table) => {
+            table.integer('line').nullable();
+        });
+    }
+    if (!(await knex.schema.hasColumn('analysis_violations', 'col'))) {
+        await knex.schema.alterTable('analysis_violations', (table) => {
+            table.integer('col').nullable();
+        });
+    }
+}
+/**
+ * Replaces the stored analysis violations with a freshly generated set.
+ *
+ * The function resolves every violation URL to a `content_items.id` (via
+ * the `url_refs` dictionary), deduplicates message/code text through
+ * `analysis_text_refs`, and rewrites `analysis_violations` in one
+ * transaction. This is the storage-side counterpart of the query-layer
+ * `getViolations` read path.
+ *
+ * Violations whose `url` still carries the pre-#225 corrupted position
+ * suffix (and that do not already carry explicit `line`/`col`) are repaired
+ * in place: {@link parseLegacyCorruptedUrl} splits the suffix off before URL
+ * resolution, so both freshly-analyzed and legacy-JSON-backfilled data end
+ * up in the same clean shape.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param violations - Flat violation list from the analyze phase.
+ */
+export async function replaceAnalysisViolations(knex, violations) {
+    await ensureLineColColumns(knex);
+    const repaired = violations.map((violation) => {
+        if (violation.line != null || violation.col != null) {
+            return violation;
+        }
+        const parsed = parseLegacyCorruptedUrl(violation.url);
+        return parsed
+            ? { ...violation, url: parsed.url, line: parsed.line, col: parsed.col }
+            : violation;
+    });
+    await knex.transaction(async (trx) => {
+        await trx('analysis_violations').delete();
+        await trx('analysis_text_refs').delete();
+        if (repaired.length === 0) {
+            return;
+        }
+        const urls = [...new Set(repaired.map((v) => v.url))];
+        const pageIdByUrl = new Map();
+        await eachSplitted(urls, 500, async (chunk) => {
+            const pageRows = await trx('content_items')
+                .join('url_refs', 'url_refs.id', 'content_items.url_id')
+                .select('content_items.id as id', 'url_refs.url as url')
+                .whereIn('url_refs.url', chunk);
+            for (const row of pageRows) {
+                pageIdByUrl.set(row.url, row.id);
+            }
+        });
+        if (pageIdByUrl.size !== urls.length) {
+            const missing = urls.filter((url) => !pageIdByUrl.has(url));
+            throw new Error(`replaceAnalysisViolations: could not resolve ${missing.length} page URL(s): ${missing[0]}`);
+        }
+        const textByValue = new Map();
+        const resolveTextId = async (text) => {
+            const cached = textByValue.get(text);
+            if (cached != null) {
+                return cached;
+            }
+            const sha256 = createHash('sha256').update(text).digest('hex');
+            await trx('analysis_text_refs')
+                .insert({ text, sha256 })
+                .onConflict(['sha256', 'text'])
+                .ignore();
+            const [existing] = await trx
+                .select('id')
+                .from('analysis_text_refs')
+                .where('sha256', sha256)
+                .where('text', text);
+            if (!existing) {
+                throw new Error(`replaceAnalysisViolations: failed to resolve text ref for ${text}`);
+            }
+            textByValue.set(text, existing.id);
+            return existing.id;
+        };
+        const rows = [];
+        for (const violation of repaired) {
+            const pageId = pageIdByUrl.get(violation.url);
+            if (!pageId) {
+                throw new Error(`replaceAnalysisViolations: could not resolve page URL: ${violation.url}`);
+            }
+            const messageTextId = await resolveTextId(violation.message);
+            const codeValue = violation.code ?? '';
+            const codeTextId = codeValue === '' ? null : await resolveTextId(codeValue);
+            rows.push({
+                page_id: pageId,
+                validator: violation.validator,
+                severity: violation.severity,
+                rule: violation.rule,
+                message_text_id: messageTextId,
+                line: violation.line ?? null,
+                col: violation.col ?? null,
+                code_text_id: codeTextId,
+                page_url_sort_key: violation.url,
+                message_sort_key: violation.message,
+                code_sort_key: codeValue,
+            });
+        }
+        await eachSplitted(rows, 500, async (chunk) => {
+            await trx('analysis_violations').insert(chunk);
+        });
+    });
+}

package/lib/archive/db-ops/anchors/get-anchors-on-page.d.ts

@@ -0,0 +1,10 @@
+import type { Knex } from 'knex';
+/**
+ * Retrieves all anchors (outgoing links) on a specific page.
+ * Joins `anchor_edges` with `content_items` (+ `page_meta` for title) to
+ * resolve link destinations.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId - The database ID of the page whose anchors to retrieve.
+ * @returns An array of anchor records with resolved URL, title, status, and content type.
+ */
+export declare function getAnchorsOnPage(knex: Knex, pageId: number): Promise<any[]>;

package/lib/archive/db-ops/anchors/get-anchors-on-page.js

@@ -0,0 +1,21 @@
+/**
+ * Retrieves all anchors (outgoing links) on a specific page.
+ * Joins `anchor_edges` with `content_items` (+ `page_meta` for title) to
+ * resolve link destinations.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId - The database ID of the page whose anchors to retrieve.
+ * @returns An array of anchor records with resolved URL, title, status, and content type.
+ */
+export async function getAnchorsOnPage(knex, pageId) {
+    const res = await knex
+        .select('dest_url.url as url', 'title_ref.text as title', 'dest.status as status', 'dest.status_text as statusText', 'ctr.raw as contentType', 'anchor_edges.first_hash as hash', 'first_text.text as textContent')
+        .from('anchor_edges')
+        .join('content_items as dest', 'anchor_edges.href_page_id', '=', 'dest.id')
+        .join('url_refs as dest_url', 'dest.url_id', '=', 'dest_url.id')
+        .leftJoin('content_type_refs as ctr', 'dest.content_type_id', '=', 'ctr.id')
+        .leftJoin('page_meta as pm', 'dest.id', '=', 'pm.page_id')
+        .leftJoin('text_refs as title_ref', 'pm.title_text_id', '=', 'title_ref.id')
+        .leftJoin('text_refs as first_text', 'anchor_edges.first_text_id', '=', 'first_text.id')
+        .where('anchor_edges.page_id', pageId);
+    return res;
+}

package/lib/archive/db-ops/config/get-base-url.d.ts

@@ -0,0 +1,8 @@
+import type { Knex } from 'knex';
+/**
+ * Retrieves the base URL of the crawl session from the `info` table.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The base URL string.
+ * @throws {Error} If no base URL is found in the database.
+ */
+export declare function getBaseUrl(knex: Knex): Promise<string>;

package/lib/archive/db-ops/config/get-base-url.js

@@ -0,0 +1,14 @@
+/**
+ * Retrieves the base URL of the crawl session from the `info` table.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The base URL string.
+ * @throws {Error} If no base URL is found in the database.
+ */
+export async function getBaseUrl(knex) {
+    const selected = await knex.select('baseUrl').from('info');
+    if (!selected[0]) {
+        throw new Error('No baseUrl');
+    }
+    const [{ baseUrl }] = selected;
+    return baseUrl || '';
+}

package/lib/archive/db-ops/config/get-config.d.ts

@@ -0,0 +1,10 @@
+import type { Config } from '../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves the full crawl configuration from the `info` table.
+ * Deserializes JSON-encoded fields (`roots`, `excludes`, `excludeKeywords`, `excludeUrls`).
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The parsed {@link Config} object.
+ * @throws {Error} If no configuration is found in the database.
+ */
+export declare function getConfig(knex: Knex): Promise<Config>;

package/lib/archive/db-ops/config/get-config.js

@@ -0,0 +1,27 @@
+import { dbLog } from '../../debug.js';
+import { getJSON } from '../../get-json.js';
+/**
+ * Retrieves the full crawl configuration from the `info` table.
+ * Deserializes JSON-encoded fields (`roots`, `excludes`, `excludeKeywords`, `excludeUrls`).
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The parsed {@link Config} object.
+ * @throws {Error} If no configuration is found in the database.
+ */
+export async function getConfig(knex) {
+    const [config] = await knex.select('*').from('info');
+    if (!config) {
+        throw new Error('No config');
+    }
+    const opt = {
+        ...config,
+        excludes: getJSON(config.excludes, []),
+        excludeKeywords: getJSON(config.excludeKeywords, []),
+        excludeUrls: getJSON(config.excludeUrls, []),
+        roots: getJSON(config.roots, []),
+        retry: config.retry ?? 3,
+    };
+    // @ts-expect-error — `id` is the primary key, not part of the public Config shape
+    delete opt.id;
+    dbLog('Table `info`: %O => %O', config, opt);
+    return opt;
+}