@nitpicker/archive 0.0.0-stage → 0.25.0

package/lib/db-ops/_shared/resolve-url-or-blob.js

@@ -0,0 +1,29 @@
+import { DATA_URI_URL_REFS_LIMIT } from '../../populate-ref-tables/data-uri-url-refs-limit.js';
+import { upsertBlobRef } from './upsert-blob-ref.js';
+import { upsertUrlRef } from './upsert-url-ref.js';
+/**
+ * Routes one URL-shaped value to either `url_refs` or `blob_refs` per the
+ * data-URI threshold rule (large data URIs live in `blob_refs`, everything
+ * else in `url_refs`), upserting into whichever dictionary applies.
+ *
+ * Shared by every writer that can receive a `data:` URI in a URL-shaped
+ * column: `image_items.src` / `current_src` and `resource_items`' own
+ * identity URL. At most one of `url` / `blob` is non-null; both are `null`
+ * when the value is empty or a malformed data URI that fails to decode.
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param value - Raw URL-shaped value.
+ * @returns `{ url, blob }` pair with at most one non-null field.
+ * @example
+ * const slot = await resolveUrlOrBlob(trx, caches, image.src);
+ * // slot.url set for a regular URL, slot.blob set for a large data: URI.
+ */
+export async function resolveUrlOrBlob(qb, caches, value) {
+    if (typeof value !== 'string' || value === '') {
+        return { url: null, blob: null };
+    }
+    if (value.startsWith('data:') && value.length > DATA_URI_URL_REFS_LIMIT) {
+        return { url: null, blob: await upsertBlobRef(qb, caches, value) };
+    }
+    return { url: await upsertUrlRef(qb, caches, value), blob: null };
+}

package/lib/db-ops/_shared/retry-setting.d.ts

@@ -0,0 +1,16 @@
+import type { RetryCallOptions } from '@d-zero/shared/retry';
+/**
+ * Default retry options shared by every `Database.*` op that wraps its
+ * knex work in {@link ../../utils/error/emit-error-with-retry.ts}
+ * (`emitErrorAndRetry`) or {@link ../../utils/error/emit-error.ts}
+ * (`emitError`, no retry). Three retries at 300 ms intervals covers the
+ * transient libsql failure modes (WAL contention, brief lock waits)
+ * without letting a genuinely broken query hang the crawler for long.
+ *
+ * Two ops deliberately do NOT use these settings and instead wrap
+ * `retryCall` themselves so they can override `label` (`getResourceByUrl`)
+ * or opt out entirely (`checkpoint`, `destroy`, `getKnex`,
+ * `setUrlOrder`, the `constructor`, and `Database.connect`). See the
+ * corresponding op files for the reasoning at each site.
+ */
+export declare const retrySetting: RetryCallOptions;

package/lib/db-ops/_shared/retry-setting.js

@@ -0,0 +1,18 @@
+/**
+ * Default retry options shared by every `Database.*` op that wraps its
+ * knex work in {@link ../../utils/error/emit-error-with-retry.ts}
+ * (`emitErrorAndRetry`) or {@link ../../utils/error/emit-error.ts}
+ * (`emitError`, no retry). Three retries at 300 ms intervals covers the
+ * transient libsql failure modes (WAL contention, brief lock waits)
+ * without letting a genuinely broken query hang the crawler for long.
+ *
+ * Two ops deliberately do NOT use these settings and instead wrap
+ * `retryCall` themselves so they can override `label` (`getResourceByUrl`)
+ * or opt out entirely (`checkpoint`, `destroy`, `getKnex`,
+ * `setUrlOrder`, the `constructor`, and `Database.connect`). See the
+ * corresponding op files for the reasoning at each site.
+ */
+export const retrySetting = {
+    interval: 300,
+    retries: 3,
+};

package/lib/db-ops/_shared/safe-parse-json.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Parses a JSON column value, returning `null` on parse failure rather
+ * than throwing. JSON columns in `page_jsonld` (`parsed`) and `page_tags`
+ * (`categories`, `sources`) are written by `JSON.stringify` and round-trip
+ * cleanly under normal conditions; a hand-edited archive that has
+ * malformed JSON in those columns should degrade gracefully rather than
+ * propagate a parse error up to the consumer.
+ * @param value - JSON-encoded text.
+ * @returns Parsed value, or `null` if the input could not be parsed.
+ */
+export declare function safeParseJson(value: string): unknown;

package/lib/db-ops/_shared/safe-parse-json.js

@@ -0,0 +1,18 @@
+/**
+ * Parses a JSON column value, returning `null` on parse failure rather
+ * than throwing. JSON columns in `page_jsonld` (`parsed`) and `page_tags`
+ * (`categories`, `sources`) are written by `JSON.stringify` and round-trip
+ * cleanly under normal conditions; a hand-edited archive that has
+ * malformed JSON in those columns should degrade gracefully rather than
+ * propagate a parse error up to the consumer.
+ * @param value - JSON-encoded text.
+ * @returns Parsed value, or `null` if the input could not be parsed.
+ */
+export function safeParseJson(value) {
+    try {
+        return JSON.parse(value);
+    }
+    catch {
+        return null;
+    }
+}

package/lib/db-ops/_shared/types.d.ts

@@ -0,0 +1,94 @@
+import type { HeaderTableCaches } from '../../populate-ref-tables/types.js';
+import type { PageSource } from '../../types.js';
+/**
+ * One cached `content_items` identity: the row id plus the provenance
+ * label at the time it was last read or written through the cache. The
+ * `source` copy lets {@link ../_shared/resolve-content-item-id.ts} apply
+ * the crawled-wins downgrade without re-reading the row on every anchor
+ * hit.
+ */
+export interface ContentItemCacheEntry {
+    /** `content_items.id`. */
+    id: number;
+    /** `content_items.source` as last observed / written by this process. */
+    source: PageSource;
+    /**
+     * `content_items.is_metadata_only` as last observed / written by this
+     * process. Lets {@link ../_shared/resolve-content-item-id.ts} skip a
+     * redundant `UPDATE` when a later resolution recomputes the same value.
+     */
+    isMetadataOnly: 0 | 1;
+}
+/**
+ * In-process id caches shared across one archive connection's write path.
+ *
+ * Every ref-table upsert primitive under `db-ops/_shared/` consults these
+ * maps before touching SQL. The caches are correct for the lifetime of a
+ * single writer connection because ref rows are append-only (`url_refs`,
+ * `text_refs`, `json_refs`, `blob_refs`, `content_type_refs`, header
+ * dictionaries never delete rows) and `content_items` identities
+ * (`id`, `url_id`) are never reassigned — only the cached `source` can
+ * change, and the sole writer of that column is the cache-aware downgrade
+ * in `resolve-content-item-id.ts` itself.
+ *
+ * NOT safe to share across processes or across two concurrent writer
+ * connections — the crawler holds a single writer connection per archive
+ * (enforced by the archive lock), so no such sharing exists.
+ */
+export interface WriteRefCaches {
+    /** `url string → url_refs.id`. */
+    readonly urlIds: Map<string, number>;
+    /** `url string → content_items` identity (id + last-known source). */
+    readonly contentItems: Map<string, ContentItemCacheEntry>;
+    /** `url string → resource_items.id`. */
+    readonly resourceIds: Map<string, number>;
+    /** `normalized content-type raw value → content_type_refs.id`. */
+    readonly contentTypeIds: Map<string, number>;
+    /** `hex(content hash) → json_refs.id`. */
+    readonly jsonIds: Map<string, number>;
+    /** `hex(content hash) → blob_refs.id`. */
+    readonly blobIds: Map<string, number>;
+    /** `hex(content hash) → console_log_items.id`. */
+    readonly consoleLogIds: Map<string, number>;
+    /**
+     * Header dictionary caches (`header_name_refs` / `header_value_refs` /
+     * `header_sets` id maps). `null` until the first header-set upsert
+     * warms them from the DB — warming issues three SELECTs, which a
+     * crawl that never stores headers (e.g. list-mode with no responses)
+     * should not pay for.
+     */
+    headers: HeaderTableCaches | null;
+}
+/**
+ * Optional per-call flags for {@link ../_shared/resolve-content-item-id.ts}.
+ */
+export interface ResolveContentItemIdOptions {
+    /**
+     * `1` marks the row as an external URL that will never be scraped as a
+     * target. Recorded on new inserts only. Defaults to `0` (in-scope) on
+     * insert, mirroring the legacy column default.
+     */
+    isExternal?: 0 | 1;
+    /**
+     * Provenance label put on a newly-inserted row. Omit to let the
+     * `content_items.source` DEFAULT (`'crawled'`) apply. Pass `'crawled'`
+     * to arm the crawled-wins downgrade on existing inventory-labelled rows.
+     */
+    source?: PageSource;
+    /**
+     * `1` marks the row as fated for a metadata-only (title-only) scrape
+     * rather than a full one, `0` explicitly marks it as a full-scrape
+     * target. Omit (the default for every caller except
+     * `replaceAnchorEdges`) to leave the column untouched — those callers
+     * (redirects, resources, errors, skipped pages, console logs) have no
+     * opinion on scrape depth and must not clobber a value an
+     * anchor-discovery call already established.
+     * `!options.recursive || anchor.isExternal` is a pure function of the
+     * URL within one crawl session (`recursive` is session-constant,
+     * `isExternal` depends only on scope matching), so every anchor-path
+     * call for the same URL always recomputes the same value — there is no
+     * real conflict to arbitrate between two *opinionated* calls, only
+     * between an opinionated call and the unopinionated majority.
+     */
+    isMetadataOnly?: 0 | 1;
+}

package/lib/db-ops/_shared/types.js

@@ -0,0 +1 @@
+export {};

package/lib/db-ops/_shared/upsert-blob-ref.d.ts

@@ -0,0 +1,25 @@
+import type { WriteRefCaches } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Resolves the `blob_refs.id` for one large `data:` URI, decoding and
+ * inserting the deduplicated (hash-keyed) payload row when it is new.
+ *
+ * Callers route values here per the data-URI threshold rule (`data:` URIs
+ * longer than `DATA_URI_URL_REFS_LIMIT` — everything else belongs in
+ * `url_refs`). The payload bytes are decoded from the base64 /
+ * percent-encoded tail, hashed, and stored zstd-compressed — the same
+ * storage contract as the rows produced when migrating an existing
+ * archive (`populate-ref-tables/populate-blob-refs.ts`).
+ *
+ * Returns `null` when the data URI fails to decode (malformed base64 /
+ * truncated percent escape). A single malformed image `src` must not
+ * abort the page write; the caller stores `*_blob_id = null` and the
+ * information loss is limited to that one slot.
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ * @param caches - The connection's write-side id caches; mutated in place.
+ * @param dataUri - The raw `data:` URI value.
+ * @returns The `blob_refs.id`, or `null` when the URI cannot be decoded.
+ * @example
+ * const id = await upsertBlobRef(knex, caches, 'data:image/png;base64,<payload>');
+ */
+export declare function upsertBlobRef(qb: Knex | Knex.Transaction, caches: WriteRefCaches, dataUri: string): Promise<number | null>;

package/lib/db-ops/_shared/upsert-blob-ref.js

@@ -0,0 +1,48 @@
+import { zstdCompressSync } from 'node:zlib';
+import { computeContentHash } from '../../populate-ref-tables/compute-content-hash.js';
+import { decodeDataUri } from '../../populate-ref-tables/decode-data-uri.js';
+/**
+ * Resolves the `blob_refs.id` for one large `data:` URI, decoding and
+ * inserting the deduplicated (hash-keyed) payload row when it is new.
+ *
+ * Callers route values here per the data-URI threshold rule (`data:` URIs
+ * longer than `DATA_URI_URL_REFS_LIMIT` — everything else belongs in
+ * `url_refs`). The payload bytes are decoded from the base64 /
+ * percent-encoded tail, hashed, and stored zstd-compressed — the same
+ * storage contract as the rows produced when migrating an existing
+ * archive (`populate-ref-tables/populate-blob-refs.ts`).
+ *
+ * Returns `null` when the data URI fails to decode (malformed base64 /
+ * truncated percent escape). A single malformed image `src` must not
+ * abort the page write; the caller stores `*_blob_id = null` and the
+ * information loss is limited to that one slot.
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ * @param caches - The connection's write-side id caches; mutated in place.
+ * @param dataUri - The raw `data:` URI value.
+ * @returns The `blob_refs.id`, or `null` when the URI cannot be decoded.
+ * @example
+ * const id = await upsertBlobRef(knex, caches, 'data:image/png;base64,<payload>');
+ */
+export async function upsertBlobRef(qb, caches, dataUri) {
+    const decoded = decodeDataUri(dataUri);
+    if (decoded === null) {
+        return null;
+    }
+    const hash = computeContentHash(decoded.bytes);
+    const hex = hash.toString('hex');
+    const cached = caches.blobIds.get(hex);
+    if (cached !== undefined) {
+        return cached;
+    }
+    const compressed = zstdCompressSync(decoded.bytes);
+    const rows = await qb.raw(`INSERT INTO blob_refs (hash, body, codec, size_raw, size_stored)
+		 VALUES (?, ?, 'zstd', ?, ?)
+		 ON CONFLICT(hash) DO UPDATE SET hash = hash
+		 RETURNING id`, [hash, compressed, decoded.bytes.byteLength, compressed.byteLength]);
+    const first = rows[0];
+    if (first === undefined) {
+        throw new Error('upsertBlobRef: RETURNING yielded no row');
+    }
+    caches.blobIds.set(hex, first.id);
+    return first.id;
+}

package/lib/db-ops/_shared/upsert-content-type-ref.d.ts

@@ -0,0 +1,30 @@
+import type { WriteRefCaches } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Resolves the `content_type_refs.id` for one content-type value,
+ * inserting the dictionary row when the value is new.
+ *
+ * `raw` is expected to be the already-canonicalised form produced by
+ * `normalizeContentType` (lower-cased, trimmed) — the write path
+ * canonicalises before storing, exactly as the legacy `pages.contentType`
+ * column did, so the dictionary's natural key never forks on case or
+ * whitespace variants of the same MIME. `normalized` is derived via the
+ * shared {@link ../../populate-ref-tables/normalize-mime.ts} (the same
+ * derivation the archive-migration populate uses, so live-crawl rows and
+ * migrated rows never fork on the same raw value); `category` via
+ * {@link ../../populate-ref-tables/classify-content-type.ts}.
+ *
+ * The content-type space is tiny (well under 1 000 distinct values on
+ * even the largest archives) so the cache converges after the first few
+ * pages and steady-state calls issue no SQL.
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ * @param caches - The connection's write-side id caches; mutated in place.
+ * @param raw - Canonicalised content-type value. Pass the output of
+ *   `normalizeContentType`; `null` / empty input must be short-circuited
+ *   by the caller (a row without a content type stores `content_type_id
+ *   = null`, not a dictionary entry).
+ * @returns The `content_type_refs.id` of the existing or new row.
+ * @example
+ * const id = await upsertContentTypeRef(knex, caches, 'text/html');
+ */
+export declare function upsertContentTypeRef(qb: Knex | Knex.Transaction, caches: WriteRefCaches, raw: string): Promise<number>;

package/lib/db-ops/_shared/upsert-content-type-ref.js

@@ -0,0 +1,45 @@
+import { classifyContentType } from '../../populate-ref-tables/classify-content-type.js';
+import { normalizeMime } from '../../populate-ref-tables/normalize-mime.js';
+/**
+ * Resolves the `content_type_refs.id` for one content-type value,
+ * inserting the dictionary row when the value is new.
+ *
+ * `raw` is expected to be the already-canonicalised form produced by
+ * `normalizeContentType` (lower-cased, trimmed) — the write path
+ * canonicalises before storing, exactly as the legacy `pages.contentType`
+ * column did, so the dictionary's natural key never forks on case or
+ * whitespace variants of the same MIME. `normalized` is derived via the
+ * shared {@link ../../populate-ref-tables/normalize-mime.ts} (the same
+ * derivation the archive-migration populate uses, so live-crawl rows and
+ * migrated rows never fork on the same raw value); `category` via
+ * {@link ../../populate-ref-tables/classify-content-type.ts}.
+ *
+ * The content-type space is tiny (well under 1 000 distinct values on
+ * even the largest archives) so the cache converges after the first few
+ * pages and steady-state calls issue no SQL.
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ * @param caches - The connection's write-side id caches; mutated in place.
+ * @param raw - Canonicalised content-type value. Pass the output of
+ *   `normalizeContentType`; `null` / empty input must be short-circuited
+ *   by the caller (a row without a content type stores `content_type_id
+ *   = null`, not a dictionary entry).
+ * @returns The `content_type_refs.id` of the existing or new row.
+ * @example
+ * const id = await upsertContentTypeRef(knex, caches, 'text/html');
+ */
+export async function upsertContentTypeRef(qb, caches, raw) {
+    const cached = caches.contentTypeIds.get(raw);
+    if (cached !== undefined) {
+        return cached;
+    }
+    const rows = await qb.raw(`INSERT INTO content_type_refs (raw, normalized, category)
+		 VALUES (?, ?, ?)
+		 ON CONFLICT(raw) DO UPDATE SET raw = raw
+		 RETURNING id`, [raw, normalizeMime(raw), classifyContentType(raw)]);
+    const first = rows[0];
+    if (first === undefined) {
+        throw new Error(`upsertContentTypeRef: RETURNING yielded no row for ${raw}`);
+    }
+    caches.contentTypeIds.set(raw, first.id);
+    return first.id;
+}

package/lib/db-ops/_shared/upsert-json-ref.d.ts

@@ -0,0 +1,22 @@
+import type { WriteRefCaches } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Resolves the `json_refs.id` for one JSON payload string, inserting the
+ * deduplicated (hash-keyed) row when the payload is new.
+ *
+ * The payload is hashed over its raw UTF-8 bytes and stored
+ * zstd-compressed (`codec = 'zstd'`) with `size_raw` / `size_stored`
+ * bookkeeping — the same storage contract as `page_html_blobs` and the
+ * dictionary rows produced when migrating an existing archive
+ * (`populate-ref-tables/populate-json-refs.ts`). Identical payloads
+ * (e.g. the same `meta_extras` JSON across templated pages) produce one
+ * row; the compression cost is paid only on a cache-and-DB miss.
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ * @param caches - The connection's write-side id caches; mutated in place.
+ * @param jsonText - Raw JSON string. Empty / null input must be
+ *   short-circuited by the caller (`*_json_id = null`).
+ * @returns The `json_refs.id` of the existing or new row.
+ * @example
+ * const id = await upsertJsonRef(knex, caches, JSON.stringify(extras));
+ */
+export declare function upsertJsonRef(qb: Knex | Knex.Transaction, caches: WriteRefCaches, jsonText: string): Promise<number>;

package/lib/db-ops/_shared/upsert-json-ref.js

@@ -0,0 +1,41 @@
+import { computeContentHash } from '../../populate-ref-tables/compute-content-hash.js';
+import { compressPayload } from './compress-payload.js';
+/**
+ * Resolves the `json_refs.id` for one JSON payload string, inserting the
+ * deduplicated (hash-keyed) row when the payload is new.
+ *
+ * The payload is hashed over its raw UTF-8 bytes and stored
+ * zstd-compressed (`codec = 'zstd'`) with `size_raw` / `size_stored`
+ * bookkeeping — the same storage contract as `page_html_blobs` and the
+ * dictionary rows produced when migrating an existing archive
+ * (`populate-ref-tables/populate-json-refs.ts`). Identical payloads
+ * (e.g. the same `meta_extras` JSON across templated pages) produce one
+ * row; the compression cost is paid only on a cache-and-DB miss.
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ * @param caches - The connection's write-side id caches; mutated in place.
+ * @param jsonText - Raw JSON string. Empty / null input must be
+ *   short-circuited by the caller (`*_json_id = null`).
+ * @returns The `json_refs.id` of the existing or new row.
+ * @example
+ * const id = await upsertJsonRef(knex, caches, JSON.stringify(extras));
+ */
+export async function upsertJsonRef(qb, caches, jsonText) {
+    const rawBytes = Buffer.from(jsonText, 'utf8');
+    const hash = computeContentHash(rawBytes);
+    const hex = hash.toString('hex');
+    const cached = caches.jsonIds.get(hex);
+    if (cached !== undefined) {
+        return cached;
+    }
+    const { body, sizeRaw, sizeStored } = compressPayload(rawBytes);
+    const rows = await qb.raw(`INSERT INTO json_refs (hash, json_text, codec, size_raw, size_stored)
+		 VALUES (?, ?, 'zstd', ?, ?)
+		 ON CONFLICT(hash) DO UPDATE SET hash = hash
+		 RETURNING id`, [hash, body, sizeRaw, sizeStored]);
+    const first = rows[0];
+    if (first === undefined) {
+        throw new Error('upsertJsonRef: RETURNING yielded no row');
+    }
+    caches.jsonIds.set(hex, first.id);
+    return first.id;
+}

package/lib/db-ops/_shared/upsert-response-headers.d.ts

@@ -0,0 +1,35 @@
+import type { WriteRefCaches } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Decomposes one response's headers and upserts them into the header
+ * dictionary tables (`header_name_refs` / `header_value_refs` /
+ * `header_sets` / `header_set_entries` / `header_flags`), returning the
+ * `header_sets.id` to store on the referring `content_items` /
+ * `resource_items` row.
+ *
+ * Runs per response (incrementally during the crawl), not deferred to
+ * crawl-end — the header dictionaries grow as pages arrive and the
+ * entity row can reference its `header_set_id` in the same write.
+ * Dedup makes the steady-state cost small: CDN-served sites reuse a
+ * handful of header shapes, so after the first few pages most calls
+ * resolve entirely from the in-process caches (warmed lazily from the DB
+ * on the first call, so a crawl that never sees headers pays nothing).
+ *
+ * Returns `null` for responses with no headers (`null` / `{}` /
+ * unparsable) — the referring row stores `header_set_id = null`, which
+ * is exactly how such rows read back through `header_flags` joins
+ * (absent flags row = no headers recorded).
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ * @param caches - The connection's write-side id caches; the header
+ *   sub-caches are created and attached on first use.
+ * @param responseHeaders - The response's header record as delivered by
+ *   the scraper (`Record<name, string | string[]>`), or `null` /
+ *   `undefined` when the response carried none.
+ * @returns The `header_sets.id`, or `null` when there is no set to store.
+ * @example
+ * const headerSetId = await upsertResponseHeaders(knex, caches, {
+ *   'content-type': 'text/html',
+ *   'cache-control': 'no-store',
+ * });
+ */
+export declare function upsertResponseHeaders(qb: Knex | Knex.Transaction, caches: WriteRefCaches, responseHeaders: Record<string, string | string[] | undefined> | null | undefined): Promise<number | null>;

package/lib/db-ops/_shared/upsert-response-headers.js

@@ -0,0 +1,49 @@
+import { createHeaderTableCaches } from '../../populate-ref-tables/create-header-table-caches.js';
+import { decomposeHeaderSet } from '../../populate-ref-tables/decompose-header-set.js';
+import { upsertOneHeaderSet } from '../../populate-ref-tables/upsert-one-header-set.js';
+/**
+ * Decomposes one response's headers and upserts them into the header
+ * dictionary tables (`header_name_refs` / `header_value_refs` /
+ * `header_sets` / `header_set_entries` / `header_flags`), returning the
+ * `header_sets.id` to store on the referring `content_items` /
+ * `resource_items` row.
+ *
+ * Runs per response (incrementally during the crawl), not deferred to
+ * crawl-end — the header dictionaries grow as pages arrive and the
+ * entity row can reference its `header_set_id` in the same write.
+ * Dedup makes the steady-state cost small: CDN-served sites reuse a
+ * handful of header shapes, so after the first few pages most calls
+ * resolve entirely from the in-process caches (warmed lazily from the DB
+ * on the first call, so a crawl that never sees headers pays nothing).
+ *
+ * Returns `null` for responses with no headers (`null` / `{}` /
+ * unparsable) — the referring row stores `header_set_id = null`, which
+ * is exactly how such rows read back through `header_flags` joins
+ * (absent flags row = no headers recorded).
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ * @param caches - The connection's write-side id caches; the header
+ *   sub-caches are created and attached on first use.
+ * @param responseHeaders - The response's header record as delivered by
+ *   the scraper (`Record<name, string | string[]>`), or `null` /
+ *   `undefined` when the response carried none.
+ * @returns The `header_sets.id`, or `null` when there is no set to store.
+ * @example
+ * const headerSetId = await upsertResponseHeaders(knex, caches, {
+ *   'content-type': 'text/html',
+ *   'cache-control': 'no-store',
+ * });
+ */
+export async function upsertResponseHeaders(qb, caches, responseHeaders) {
+    // Serialise exactly like the scraper's own JSON storage form so the
+    // decomposition (and its raw_json_hash dedup key) matches what a bulk
+    // archive populate would compute for the same response.
+    const rawJson = responseHeaders == null ? null : JSON.stringify(responseHeaders);
+    const decomposed = decomposeHeaderSet(rawJson);
+    if (decomposed === null) {
+        return null;
+    }
+    if (caches.headers === null) {
+        caches.headers = await createHeaderTableCaches(qb);
+    }
+    return await upsertOneHeaderSet(qb, decomposed, caches.headers);
+}

package/lib/db-ops/_shared/upsert-url-ref.d.ts

@@ -0,0 +1,39 @@
+import type { WriteRefCaches } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Resolves the `url_refs.id` for one URL string, inserting the row (with
+ * its decomposed `scheme` / `host` / `port` / `path` / `query_hash` /
+ * `fragment` columns) when the URL is not yet in the dictionary.
+ *
+ * The write path calls this for every URL it touches — page identities,
+ * anchor hrefs, image `src` / `currentSrc`, resource URLs, and URL-shaped
+ * meta columns — so the result is cached in `caches.urlIds` and a cache
+ * hit issues no SQL at all. Anchor hrefs and CDN-hosted image URLs repeat
+ * across thousands of pages, so the hit rate dominates after the first
+ * few pages.
+ *
+ * The cache-miss upsert is a single statement:
+ *
+ * ```sql
+ * INSERT INTO url_refs (...) VALUES (...)
+ * ON CONFLICT(url) DO UPDATE SET url = url
+ * RETURNING id
+ * ```
+ *
+ * `DO UPDATE SET url = url` is a deliberate no-op update whose only
+ * purpose is to make `RETURNING id` yield the existing row's id on
+ * conflict. `INSERT OR IGNORE ... RETURNING id` must NOT be used here:
+ * `RETURNING` returns zero rows when the insert is ignored, which would
+ * leave the caller with no id and a `null` FK on the next entity INSERT.
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ *   Callers inside a transaction MUST pass the `trx` so the read sees
+ *   uncommitted rows from the same transaction.
+ * @param caches - The connection's write-side id caches; mutated in place.
+ * @param url - The URL string to resolve. Stored verbatim as the natural key.
+ * @returns The `url_refs.id` of the existing or newly inserted row.
+ * @example
+ * const id = await upsertUrlRef(knex, caches, 'https://example.com/a');
+ * const again = await upsertUrlRef(knex, caches, 'https://example.com/a');
+ * // id === again, second call hits the cache and issues no SQL
+ */
+export declare function upsertUrlRef(qb: Knex | Knex.Transaction, caches: WriteRefCaches, url: string): Promise<number>;

package/lib/db-ops/_shared/upsert-url-ref.js

@@ -0,0 +1,62 @@
+import { decomposeUrl } from '../../populate-ref-tables/decompose-url.js';
+/**
+ * Resolves the `url_refs.id` for one URL string, inserting the row (with
+ * its decomposed `scheme` / `host` / `port` / `path` / `query_hash` /
+ * `fragment` columns) when the URL is not yet in the dictionary.
+ *
+ * The write path calls this for every URL it touches — page identities,
+ * anchor hrefs, image `src` / `currentSrc`, resource URLs, and URL-shaped
+ * meta columns — so the result is cached in `caches.urlIds` and a cache
+ * hit issues no SQL at all. Anchor hrefs and CDN-hosted image URLs repeat
+ * across thousands of pages, so the hit rate dominates after the first
+ * few pages.
+ *
+ * The cache-miss upsert is a single statement:
+ *
+ * ```sql
+ * INSERT INTO url_refs (...) VALUES (...)
+ * ON CONFLICT(url) DO UPDATE SET url = url
+ * RETURNING id
+ * ```
+ *
+ * `DO UPDATE SET url = url` is a deliberate no-op update whose only
+ * purpose is to make `RETURNING id` yield the existing row's id on
+ * conflict. `INSERT OR IGNORE ... RETURNING id` must NOT be used here:
+ * `RETURNING` returns zero rows when the insert is ignored, which would
+ * leave the caller with no id and a `null` FK on the next entity INSERT.
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ *   Callers inside a transaction MUST pass the `trx` so the read sees
+ *   uncommitted rows from the same transaction.
+ * @param caches - The connection's write-side id caches; mutated in place.
+ * @param url - The URL string to resolve. Stored verbatim as the natural key.
+ * @returns The `url_refs.id` of the existing or newly inserted row.
+ * @example
+ * const id = await upsertUrlRef(knex, caches, 'https://example.com/a');
+ * const again = await upsertUrlRef(knex, caches, 'https://example.com/a');
+ * // id === again, second call hits the cache and issues no SQL
+ */
+export async function upsertUrlRef(qb, caches, url) {
+    const cached = caches.urlIds.get(url);
+    if (cached !== undefined) {
+        return cached;
+    }
+    const decomposed = decomposeUrl(url);
+    const rows = await qb.raw(`INSERT INTO url_refs (url, scheme, host, port, path, query_hash, fragment)
+		 VALUES (?, ?, ?, ?, ?, ?, ?)
+		 ON CONFLICT(url) DO UPDATE SET url = url
+		 RETURNING id`, [
+        url,
+        decomposed.scheme,
+        decomposed.host,
+        decomposed.port,
+        decomposed.path,
+        decomposed.query_hash,
+        decomposed.fragment,
+    ]);
+    const first = rows[0];
+    if (first === undefined) {
+        throw new Error(`upsertUrlRef: RETURNING yielded no row for ${url}`);
+    }
+    caches.urlIds.set(url, first.id);
+    return first.id;
+}

package/lib/db-ops/anchors/get-anchors-on-page.d.ts

@@ -0,0 +1,10 @@
+import type { Knex } from 'knex';
+/**
+ * Retrieves all anchors (outgoing links) on a specific page.
+ * Joins `anchor_edges` with `content_items` (+ `page_meta` for title) to
+ * resolve link destinations.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId - The database ID of the page whose anchors to retrieve.
+ * @returns An array of anchor records with resolved URL, title, status, and content type.
+ */
+export declare function getAnchorsOnPage(knex: Knex, pageId: number): Promise<any[]>;

package/lib/db-ops/anchors/get-anchors-on-page.js

@@ -0,0 +1,21 @@
+/**
+ * Retrieves all anchors (outgoing links) on a specific page.
+ * Joins `anchor_edges` with `content_items` (+ `page_meta` for title) to
+ * resolve link destinations.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId - The database ID of the page whose anchors to retrieve.
+ * @returns An array of anchor records with resolved URL, title, status, and content type.
+ */
+export async function getAnchorsOnPage(knex, pageId) {
+    const res = await knex
+        .select('dest_url.url as url', 'title_ref.text as title', 'dest.status as status', 'dest.status_text as statusText', 'ctr.raw as contentType', 'anchor_edges.first_hash as hash', 'first_text.text as textContent')
+        .from('anchor_edges')
+        .join('content_items as dest', 'anchor_edges.href_page_id', '=', 'dest.id')
+        .join('url_refs as dest_url', 'dest.url_id', '=', 'dest_url.id')
+        .leftJoin('content_type_refs as ctr', 'dest.content_type_id', '=', 'ctr.id')
+        .leftJoin('page_meta as pm', 'dest.id', '=', 'pm.page_id')
+        .leftJoin('text_refs as title_ref', 'pm.title_text_id', '=', 'title_ref.id')
+        .leftJoin('text_refs as first_text', 'anchor_edges.first_text_id', '=', 'first_text.id')
+        .where('anchor_edges.page_id', pageId);
+    return res;
+}

package/lib/db-ops/config/get-base-url.d.ts

@@ -0,0 +1,8 @@
+import type { Knex } from 'knex';
+/**
+ * Retrieves the base URL of the crawl session from the `info` table.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The base URL string.
+ * @throws {Error} If no base URL is found in the database.
+ */
+export declare function getBaseUrl(knex: Knex): Promise<string>;