@nitpicker/crawler 0.11.0 → 0.13.0

package/lib/archive/db-ops/_shared/clear-write-ref-caches.d.ts

@@ -0,0 +1,27 @@
+import type { WriteRefCaches } from './types.js';
+/**
+ * Empties every map in a {@link WriteRefCaches} bundle in place.
+ *
+ * Called when a write transaction that used the caches rolls back. SQLite
+ * `ROLLBACK` undoes every row the transaction inserted, but `AUTOINCREMENT`
+ * counters never rewind — so an id cached optimistically during the failed
+ * attempt would survive the rollback and no longer correspond to any row.
+ * A retried attempt reusing that id would issue `UPDATE ... WHERE id = ?`
+ * against a nonexistent row and silently affect zero rows instead of
+ * failing loudly.
+ *
+ * A full clear (rather than tracking which entries the failed transaction
+ * touched) is the simplest correct fix: transient write failures are rare
+ * (retry exists precisely because they are), so paying a one-time
+ * DB-again warm-up after a rollback is cheap next to the correctness risk
+ * of a partially-poisoned cache.
+ * @param caches - The cache bundle to clear.
+ * @example
+ * try {
+ *   await knex.transaction(async (trx) => { ... });
+ * } catch (error) {
+ *   clearWriteRefCaches(caches);
+ *   throw error;
+ * }
+ */
+export declare function clearWriteRefCaches(caches: WriteRefCaches): void;

package/lib/archive/db-ops/_shared/clear-write-ref-caches.js

@@ -0,0 +1,34 @@
+/**
+ * Empties every map in a {@link WriteRefCaches} bundle in place.
+ *
+ * Called when a write transaction that used the caches rolls back. SQLite
+ * `ROLLBACK` undoes every row the transaction inserted, but `AUTOINCREMENT`
+ * counters never rewind — so an id cached optimistically during the failed
+ * attempt would survive the rollback and no longer correspond to any row.
+ * A retried attempt reusing that id would issue `UPDATE ... WHERE id = ?`
+ * against a nonexistent row and silently affect zero rows instead of
+ * failing loudly.
+ *
+ * A full clear (rather than tracking which entries the failed transaction
+ * touched) is the simplest correct fix: transient write failures are rare
+ * (retry exists precisely because they are), so paying a one-time
+ * DB-again warm-up after a rollback is cheap next to the correctness risk
+ * of a partially-poisoned cache.
+ * @param caches - The cache bundle to clear.
+ * @example
+ * try {
+ *   await knex.transaction(async (trx) => { ... });
+ * } catch (error) {
+ *   clearWriteRefCaches(caches);
+ *   throw error;
+ * }
+ */
+export function clearWriteRefCaches(caches) {
+    caches.urlIds.clear();
+    caches.contentItems.clear();
+    caches.resourceIds.clear();
+    caches.contentTypeIds.clear();
+    caches.jsonIds.clear();
+    caches.blobIds.clear();
+    caches.headers = null;
+}

package/lib/archive/db-ops/_shared/create-write-ref-caches.d.ts

@@ -0,0 +1,17 @@
+import type { WriteRefCaches } from './types.js';
+/**
+ * Creates an empty {@link WriteRefCaches} bundle for one archive writer
+ * connection.
+ *
+ * All maps start empty and fill lazily as the write path touches URLs,
+ * content types, JSON payloads, and data-URI blobs. The header caches
+ * (`headers`) start as `null` and are warmed from the DB by the first
+ * header-set upsert — see {@link ./types.ts} for the cache-correctness
+ * argument (append-only ref tables + single writer connection).
+ * @returns A fresh cache bundle. One per `Database` instance; never share
+ *   across connections.
+ * @example
+ * const caches = createWriteRefCaches();
+ * const urlId = await upsertUrlRef(knex, caches, 'https://example.com/');
+ */
+export declare function createWriteRefCaches(): WriteRefCaches;

package/lib/archive/db-ops/_shared/create-write-ref-caches.js

@@ -0,0 +1,26 @@
+/**
+ * Creates an empty {@link WriteRefCaches} bundle for one archive writer
+ * connection.
+ *
+ * All maps start empty and fill lazily as the write path touches URLs,
+ * content types, JSON payloads, and data-URI blobs. The header caches
+ * (`headers`) start as `null` and are warmed from the DB by the first
+ * header-set upsert — see {@link ./types.ts} for the cache-correctness
+ * argument (append-only ref tables + single writer connection).
+ * @returns A fresh cache bundle. One per `Database` instance; never share
+ *   across connections.
+ * @example
+ * const caches = createWriteRefCaches();
+ * const urlId = await upsertUrlRef(knex, caches, 'https://example.com/');
+ */
+export function createWriteRefCaches() {
+    return {
+        urlIds: new Map(),
+        contentItems: new Map(),
+        resourceIds: new Map(),
+        contentTypeIds: new Map(),
+        jsonIds: new Map(),
+        blobIds: new Map(),
+        headers: null,
+    };
+}

package/lib/archive/db-ops/_shared/decode-json-ref.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Decodes one `json_refs` payload back into its JSON string.
+ *
+ * The write path (`upsert-json-ref.ts`) and the archive-migration
+ * populate (`populate-json-refs.ts`) both store bodies zstd-compressed;
+ * the `'none'` codec is accepted because the `json_refs.codec` column
+ * permits it, not because either writer currently produces it. Corrupt
+ * bodies fail closed to `null` rather than throwing — readers treat an
+ * undecodable payload the same as an absent one.
+ * @param body - The raw `json_refs.json_text` body, or null when absent.
+ * @param codec - The `json_refs.codec` value, or null when absent.
+ * @returns The decoded JSON string, or null when there is no body or the
+ *   body cannot be decoded.
+ * @example
+ * const metaExtras = decodeJsonRef(row.extras_body, row.extras_codec);
+ */
+export declare function decodeJsonRef(body: Buffer | string | null, codec: 'zstd' | 'none' | null): string | null;

package/lib/archive/db-ops/_shared/decode-json-ref.js

@@ -0,0 +1,31 @@
+import { zstdDecompressSync } from 'node:zlib';
+/**
+ * Decodes one `json_refs` payload back into its JSON string.
+ *
+ * The write path (`upsert-json-ref.ts`) and the archive-migration
+ * populate (`populate-json-refs.ts`) both store bodies zstd-compressed;
+ * the `'none'` codec is accepted because the `json_refs.codec` column
+ * permits it, not because either writer currently produces it. Corrupt
+ * bodies fail closed to `null` rather than throwing — readers treat an
+ * undecodable payload the same as an absent one.
+ * @param body - The raw `json_refs.json_text` body, or null when absent.
+ * @param codec - The `json_refs.codec` value, or null when absent.
+ * @returns The decoded JSON string, or null when there is no body or the
+ *   body cannot be decoded.
+ * @example
+ * const metaExtras = decodeJsonRef(row.extras_body, row.extras_codec);
+ */
+export function decodeJsonRef(body, codec) {
+    if (body == null) {
+        return null;
+    }
+    try {
+        if (codec === 'zstd') {
+            return zstdDecompressSync(body).toString('utf8');
+        }
+        return typeof body === 'string' ? body : body.toString('utf8');
+    }
+    catch {
+        return null;
+    }
+}

package/lib/archive/db-ops/_shared/load-response-headers-by-set-ids.d.ts

@@ -0,0 +1,20 @@
+import type { Knex } from 'knex';
+/**
+ * Loads and merges the response headers for every given `header_sets.id`
+ * back into flat `Record<name, value>` objects, in one chunked batch
+ * (never one query per row).
+ *
+ * Multi-value headers (repeated `Set-Cookie` etc.) are joined with
+ * `', '` in `occurrence` order within each name — the flat-record shape
+ * the pre-0.13 `responseHeaders` JSON column stored, which every reader
+ * of the reconstructed rows still expects.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param headerSetIds - Distinct `header_sets.id` values to load. Empty
+ *   input returns an empty map without touching the DB.
+ * @returns Map from `header_sets.id` to the merged header record. Ids
+ *   with no entries are absent from the map.
+ * @example
+ * const headersBySetId = await loadResponseHeadersBySetIds(knex, [1, 2]);
+ * const record = headersBySetId.get(1) ?? {};
+ */
+export declare function loadResponseHeadersBySetIds(knex: Knex, headerSetIds: readonly number[]): Promise<Map<number, Record<string, string>>>;

package/lib/archive/db-ops/_shared/load-response-headers-by-set-ids.js

@@ -0,0 +1,53 @@
+/**
+ * Ids per `WHERE header_set_id IN (?, ...)` chunk. Modern SQLite builds
+ * (including the vendored libsql) reject statements binding more than
+ * 32,766 parameters, and header sets dedupe poorly on real sites (Date /
+ * ETag / Set-Cookie make most sets unique), so an IN-list sized by the
+ * page batch without chunking would exceed the limit on large archives.
+ * 800 matches
+ * the `LOOKUP_CHUNK_SIZE` convention the populate/upsert paths already
+ * use (`resolve-text-refs.ts` and siblings).
+ */
+const LOOKUP_CHUNK_SIZE = 800;
+/**
+ * Loads and merges the response headers for every given `header_sets.id`
+ * back into flat `Record<name, value>` objects, in one chunked batch
+ * (never one query per row).
+ *
+ * Multi-value headers (repeated `Set-Cookie` etc.) are joined with
+ * `', '` in `occurrence` order within each name — the flat-record shape
+ * the pre-0.13 `responseHeaders` JSON column stored, which every reader
+ * of the reconstructed rows still expects.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param headerSetIds - Distinct `header_sets.id` values to load. Empty
+ *   input returns an empty map without touching the DB.
+ * @returns Map from `header_sets.id` to the merged header record. Ids
+ *   with no entries are absent from the map.
+ * @example
+ * const headersBySetId = await loadResponseHeadersBySetIds(knex, [1, 2]);
+ * const record = headersBySetId.get(1) ?? {};
+ */
+export async function loadResponseHeadersBySetIds(knex, headerSetIds) {
+    const merged = new Map();
+    for (let index = 0; index < headerSetIds.length; index += LOOKUP_CHUNK_SIZE) {
+        const chunk = headerSetIds.slice(index, index + LOOKUP_CHUNK_SIZE);
+        const headerRows = (await knex('header_set_entries as hse')
+            .join('header_name_refs as hnr', 'hnr.id', 'hse.name_id')
+            .join('header_value_refs as hvr', 'hvr.id', 'hse.value_id')
+            .whereIn('hse.header_set_id', chunk)
+            .orderBy(['hse.header_set_id', 'hnr.name', 'hse.occurrence'])
+            .select('hse.header_set_id as headerSetId', 'hnr.name as name', 'hvr.value as value'));
+        for (const row of headerRows) {
+            const bySet = merged.get(row.headerSetId) ?? new Map();
+            const values = bySet.get(row.name) ?? [];
+            values.push(row.value);
+            bySet.set(row.name, values);
+            merged.set(row.headerSetId, bySet);
+        }
+    }
+    const result = new Map();
+    for (const [setId, bySet] of merged) {
+        result.set(setId, Object.fromEntries([...bySet.entries()].map(([k, v]) => [k, v.join(', ')])));
+    }
+    return result;
+}

package/lib/archive/db-ops/_shared/resolve-content-item-id.d.ts

@@ -0,0 +1,61 @@
+import type { WriteRefCaches } from './types.js';
+import type { PageSource } from '../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Returns the `content_items.id` for `url`, inserting a placeholder row
+ * (via `url_refs`) when the URL is not yet known. Foundational identity
+ * primitive shared by every op that needs to record an edge or lookup to
+ * a URL — the `content_items` successor of the legacy `pages.id`
+ * resolver.
+ *
+ * **Source semantics.** `source` is written ONLY on the INSERT path —
+ * when the row already exists, the INSERT is never reached and the
+ * existing row's `source` stays untouched. This is what keeps a second
+ * `crawl --inventory` from "demoting" a page that was first labelled
+ * `'inventory-seed'` back to `'inventory-discovered'` on later passes.
+ *
+ * **Crawled-wins downgrade.** When a row that was previously labelled
+ * `'inventory-seed'` or `'inventory-discovered'` is re-encountered via a
+ * `'crawled'`-lineage anchor (the parent page is part of the graph
+ * reachable from the original crawl roots), the row is downgraded to
+ * `'crawled'`. The inventory goal is finding orphans — anything reachable
+ * from the crawled chain is NOT an orphan and should not retain an
+ * inventory label. The cached `source` is updated in the same step so a
+ * later hit does not re-issue the UPDATE.
+ *
+ * **Cache poisoning on rollback.** Entries cached inside a transaction
+ * that later rolls back would point at ids that no longer exist. Every
+ * write op that opens a multi-statement transaction around this function
+ * must discard the whole cache bundle when that transaction fails,
+ * before any retry — see the try/catch + `clearWriteRefCaches` wrappers
+ * in `update-page.ts` and `record-redirect.ts`.
+ *
+ * **Race-condition safety.** The INSERT uses `ON CONFLICT(url_id) DO
+ * UPDATE SET url_id = url_id RETURNING id, source` — the same no-op
+ * -update idiom as `upsertUrlRef` — so when a concurrent transaction
+ * wins the insert race, the statement still returns the existing row's
+ * id and source in one round trip. `onConflict().ignore()` without
+ * `RETURNING` must NOT be used here: knex's sqlite-family dialects
+ * report the connection's stale `lastInsertRowid` for a conflict-ignored
+ * insert, which reads as a valid id belonging to an unrelated row.
+ * @param qb - Knex instance OR a transaction, used verbatim for every
+ *   statement this function issues. Pass the `trx` when running inside a
+ *   transaction so reads see uncommitted writes from that transaction.
+ * @param caches - The connection's write-side id caches; mutated in place.
+ * @param url - The URL to look up or insert (normalised
+ *   `withoutHashAndAuth` form, matching the legacy identity contract).
+ * @param isExternal - Optional; recorded on new inserts only. `1` marks
+ *   the row as an external URL that will never be scraped as a target.
+ *   Defaults to `0` (in-scope) on insert, mirroring the legacy column
+ *   default.
+ * @param source - Optional provenance label put on a newly-inserted row.
+ *   Omit to let the `content_items.source` DEFAULT (`'crawled'`) apply.
+ *   Pass `'crawled'` to arm the crawled-wins downgrade on existing
+ *   inventory-labelled rows.
+ * @returns The `content_items.id` of the existing or newly inserted row.
+ * @throws {Error} When the upsert's `RETURNING` yields no row — should
+ *   not happen, so it surfaces as a hard error.
+ * @example
+ * const pageId = await resolveContentItemId(trx, caches, anchor.href, 1, 'crawled');
+ */
+export declare function resolveContentItemId(qb: Knex | Knex.Transaction, caches: WriteRefCaches, url: string, isExternal?: 0 | 1, source?: PageSource): Promise<number>;

package/lib/archive/db-ops/_shared/resolve-content-item-id.js

@@ -0,0 +1,111 @@
+import { upsertUrlRef } from './upsert-url-ref.js';
+/**
+ * Returns the `content_items.id` for `url`, inserting a placeholder row
+ * (via `url_refs`) when the URL is not yet known. Foundational identity
+ * primitive shared by every op that needs to record an edge or lookup to
+ * a URL — the `content_items` successor of the legacy `pages.id`
+ * resolver.
+ *
+ * **Source semantics.** `source` is written ONLY on the INSERT path —
+ * when the row already exists, the INSERT is never reached and the
+ * existing row's `source` stays untouched. This is what keeps a second
+ * `crawl --inventory` from "demoting" a page that was first labelled
+ * `'inventory-seed'` back to `'inventory-discovered'` on later passes.
+ *
+ * **Crawled-wins downgrade.** When a row that was previously labelled
+ * `'inventory-seed'` or `'inventory-discovered'` is re-encountered via a
+ * `'crawled'`-lineage anchor (the parent page is part of the graph
+ * reachable from the original crawl roots), the row is downgraded to
+ * `'crawled'`. The inventory goal is finding orphans — anything reachable
+ * from the crawled chain is NOT an orphan and should not retain an
+ * inventory label. The cached `source` is updated in the same step so a
+ * later hit does not re-issue the UPDATE.
+ *
+ * **Cache poisoning on rollback.** Entries cached inside a transaction
+ * that later rolls back would point at ids that no longer exist. Every
+ * write op that opens a multi-statement transaction around this function
+ * must discard the whole cache bundle when that transaction fails,
+ * before any retry — see the try/catch + `clearWriteRefCaches` wrappers
+ * in `update-page.ts` and `record-redirect.ts`.
+ *
+ * **Race-condition safety.** The INSERT uses `ON CONFLICT(url_id) DO
+ * UPDATE SET url_id = url_id RETURNING id, source` — the same no-op
+ * -update idiom as `upsertUrlRef` — so when a concurrent transaction
+ * wins the insert race, the statement still returns the existing row's
+ * id and source in one round trip. `onConflict().ignore()` without
+ * `RETURNING` must NOT be used here: knex's sqlite-family dialects
+ * report the connection's stale `lastInsertRowid` for a conflict-ignored
+ * insert, which reads as a valid id belonging to an unrelated row.
+ * @param qb - Knex instance OR a transaction, used verbatim for every
+ *   statement this function issues. Pass the `trx` when running inside a
+ *   transaction so reads see uncommitted writes from that transaction.
+ * @param caches - The connection's write-side id caches; mutated in place.
+ * @param url - The URL to look up or insert (normalised
+ *   `withoutHashAndAuth` form, matching the legacy identity contract).
+ * @param isExternal - Optional; recorded on new inserts only. `1` marks
+ *   the row as an external URL that will never be scraped as a target.
+ *   Defaults to `0` (in-scope) on insert, mirroring the legacy column
+ *   default.
+ * @param source - Optional provenance label put on a newly-inserted row.
+ *   Omit to let the `content_items.source` DEFAULT (`'crawled'`) apply.
+ *   Pass `'crawled'` to arm the crawled-wins downgrade on existing
+ *   inventory-labelled rows.
+ * @returns The `content_items.id` of the existing or newly inserted row.
+ * @throws {Error} When the upsert's `RETURNING` yields no row — should
+ *   not happen, so it surfaces as a hard error.
+ * @example
+ * const pageId = await resolveContentItemId(trx, caches, anchor.href, 1, 'crawled');
+ */
+export async function resolveContentItemId(qb, caches, url, isExternal, source) {
+    const cached = caches.contentItems.get(url);
+    if (cached !== undefined) {
+        await applyCrawledWinsDowngrade(qb, cached, source);
+        return cached.id;
+    }
+    const urlId = await upsertUrlRef(qb, caches, url);
+    const [record] = (await qb
+        .select('id', 'source')
+        .from('content_items')
+        .where('url_id', urlId));
+    if (record !== undefined) {
+        const entry = { id: record.id, source: record.source };
+        await applyCrawledWinsDowngrade(qb, entry, source);
+        caches.contentItems.set(url, entry);
+        return entry.id;
+    }
+    const insertedRows = await qb.raw(`INSERT INTO content_items (url_id, scraped, is_target, is_external${source === undefined ? '' : ', source'})
+		 VALUES (?, 0, 0, ?${source === undefined ? '' : ', ?'})
+		 ON CONFLICT(url_id) DO UPDATE SET url_id = url_id
+		 RETURNING id, source`, source === undefined ? [urlId, isExternal ?? 0] : [urlId, isExternal ?? 0, source]);
+    const inserted = insertedRows[0];
+    if (inserted === undefined) {
+        throw new Error(`Failed to insert a new content item: ${url}`);
+    }
+    const insertedEntry = {
+        id: inserted.id,
+        source: inserted.source,
+    };
+    // A conflict means a concurrent writer created the row between this
+    // function's SELECT miss and the INSERT — the returned `source` is that
+    // row's value, so the downgrade must be evaluated exactly as on the
+    // SELECT-hit path.
+    await applyCrawledWinsDowngrade(qb, insertedEntry, source);
+    caches.contentItems.set(url, insertedEntry);
+    return insertedEntry.id;
+}
+/**
+ * Fires the crawled-wins downgrade when a `'crawled'`-lineage resolution
+ * lands on a row whose last-known `source` is an inventory label, and
+ * keeps the cache entry in sync so the UPDATE runs at most once per
+ * (connection, row).
+ * @param qb - Knex instance or transaction.
+ * @param entry - The cached identity to check and mutate.
+ * @param source - The resolution's lineage label.
+ */
+async function applyCrawledWinsDowngrade(qb, entry, source) {
+    if (source !== 'crawled' || entry.source === 'crawled') {
+        return;
+    }
+    await qb('content_items').where('id', entry.id).update({ source: 'crawled' });
+    entry.source = 'crawled';
+}

package/lib/archive/db-ops/_shared/resolve-url-or-blob.d.ts

@@ -0,0 +1,23 @@
+import type { WriteRefCaches } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Routes one URL-shaped value to either `url_refs` or `blob_refs` per the
+ * data-URI threshold rule (large data URIs live in `blob_refs`, everything
+ * else in `url_refs`), upserting into whichever dictionary applies.
+ *
+ * Shared by every writer that can receive a `data:` URI in a URL-shaped
+ * column: `image_items.src` / `current_src` and `resource_items`' own
+ * identity URL. At most one of `url` / `blob` is non-null; both are `null`
+ * when the value is empty or a malformed data URI that fails to decode.
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param value - Raw URL-shaped value.
+ * @returns `{ url, blob }` pair with at most one non-null field.
+ * @example
+ * const slot = await resolveUrlOrBlob(trx, caches, image.src);
+ * // slot.url set for a regular URL, slot.blob set for a large data: URI.
+ */
+export declare function resolveUrlOrBlob(qb: Knex | Knex.Transaction, caches: WriteRefCaches, value: string | null | undefined): Promise<{
+    url: number | null;
+    blob: number | null;
+}>;

package/lib/archive/db-ops/_shared/resolve-url-or-blob.js

@@ -0,0 +1,29 @@
+import { DATA_URI_URL_REFS_LIMIT } from '../../populate-ref-tables/data-uri-url-refs-limit.js';
+import { upsertBlobRef } from './upsert-blob-ref.js';
+import { upsertUrlRef } from './upsert-url-ref.js';
+/**
+ * Routes one URL-shaped value to either `url_refs` or `blob_refs` per the
+ * data-URI threshold rule (large data URIs live in `blob_refs`, everything
+ * else in `url_refs`), upserting into whichever dictionary applies.
+ *
+ * Shared by every writer that can receive a `data:` URI in a URL-shaped
+ * column: `image_items.src` / `current_src` and `resource_items`' own
+ * identity URL. At most one of `url` / `blob` is non-null; both are `null`
+ * when the value is empty or a malformed data URI that fails to decode.
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param value - Raw URL-shaped value.
+ * @returns `{ url, blob }` pair with at most one non-null field.
+ * @example
+ * const slot = await resolveUrlOrBlob(trx, caches, image.src);
+ * // slot.url set for a regular URL, slot.blob set for a large data: URI.
+ */
+export async function resolveUrlOrBlob(qb, caches, value) {
+    if (typeof value !== 'string' || value === '') {
+        return { url: null, blob: null };
+    }
+    if (value.startsWith('data:') && value.length > DATA_URI_URL_REFS_LIMIT) {
+        return { url: null, blob: await upsertBlobRef(qb, caches, value) };
+    }
+    return { url: await upsertUrlRef(qb, caches, value), blob: null };
+}

package/lib/archive/db-ops/_shared/retry-setting.d.ts

@@ -0,0 +1,16 @@
+import type { RetryCallOptions } from '@d-zero/shared/retry';
+/**
+ * Default retry options shared by every `Database.*` op that wraps its
+ * knex work in {@link ../../../utils/error/emit-error-with-retry.ts}
+ * (`emitErrorAndRetry`) or {@link ../../../utils/error/emit-error.ts}
+ * (`emitError`, no retry). Three retries at 300 ms intervals covers the
+ * transient libsql failure modes (WAL contention, brief lock waits)
+ * without letting a genuinely broken query hang the crawler for long.
+ *
+ * Two ops deliberately do NOT use these settings and instead wrap
+ * `retryCall` themselves so they can override `label` (`getResourceByUrl`)
+ * or opt out entirely (`checkpoint`, `destroy`, `getKnex`,
+ * `setUrlOrder`, the `constructor`, and `Database.connect`). See the
+ * corresponding op files for the reasoning at each site.
+ */
+export declare const retrySetting: RetryCallOptions;

package/lib/archive/db-ops/_shared/retry-setting.js

@@ -0,0 +1,18 @@
+/**
+ * Default retry options shared by every `Database.*` op that wraps its
+ * knex work in {@link ../../../utils/error/emit-error-with-retry.ts}
+ * (`emitErrorAndRetry`) or {@link ../../../utils/error/emit-error.ts}
+ * (`emitError`, no retry). Three retries at 300 ms intervals covers the
+ * transient libsql failure modes (WAL contention, brief lock waits)
+ * without letting a genuinely broken query hang the crawler for long.
+ *
+ * Two ops deliberately do NOT use these settings and instead wrap
+ * `retryCall` themselves so they can override `label` (`getResourceByUrl`)
+ * or opt out entirely (`checkpoint`, `destroy`, `getKnex`,
+ * `setUrlOrder`, the `constructor`, and `Database.connect`). See the
+ * corresponding op files for the reasoning at each site.
+ */
+export const retrySetting = {
+    interval: 300,
+    retries: 3,
+};

package/lib/archive/db-ops/_shared/safe-parse-json.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Parses a JSON column value, returning `null` on parse failure rather
+ * than throwing. JSON columns in `page_jsonld` (`parsed`) and `page_tags`
+ * (`categories`, `sources`) are written by `JSON.stringify` and round-trip
+ * cleanly under normal conditions; a hand-edited archive that has
+ * malformed JSON in those columns should degrade gracefully rather than
+ * propagate a parse error up to the consumer.
+ * @param value - JSON-encoded text.
+ * @returns Parsed value, or `null` if the input could not be parsed.
+ */
+export declare function safeParseJson(value: string): unknown;

package/lib/archive/db-ops/_shared/safe-parse-json.js

@@ -0,0 +1,18 @@
+/**
+ * Parses a JSON column value, returning `null` on parse failure rather
+ * than throwing. JSON columns in `page_jsonld` (`parsed`) and `page_tags`
+ * (`categories`, `sources`) are written by `JSON.stringify` and round-trip
+ * cleanly under normal conditions; a hand-edited archive that has
+ * malformed JSON in those columns should degrade gracefully rather than
+ * propagate a parse error up to the consumer.
+ * @param value - JSON-encoded text.
+ * @returns Parsed value, or `null` if the input could not be parsed.
+ */
+export function safeParseJson(value) {
+    try {
+        return JSON.parse(value);
+    }
+    catch {
+        return null;
+    }
+}

package/lib/archive/db-ops/_shared/types.d.ts

@@ -0,0 +1,53 @@
+import type { HeaderTableCaches } from '../../populate-ref-tables/types.js';
+import type { PageSource } from '../../types.js';
+/**
+ * One cached `content_items` identity: the row id plus the provenance
+ * label at the time it was last read or written through the cache. The
+ * `source` copy lets {@link ../_shared/resolve-content-item-id.ts} apply
+ * the crawled-wins downgrade without re-reading the row on every anchor
+ * hit.
+ */
+export interface ContentItemCacheEntry {
+    /** `content_items.id`. */
+    id: number;
+    /** `content_items.source` as last observed / written by this process. */
+    source: PageSource;
+}
+/**
+ * In-process id caches shared across one archive connection's write path.
+ *
+ * Every ref-table upsert primitive under `db-ops/_shared/` consults these
+ * maps before touching SQL. The caches are correct for the lifetime of a
+ * single writer connection because ref rows are append-only (`url_refs`,
+ * `text_refs`, `json_refs`, `blob_refs`, `content_type_refs`, header
+ * dictionaries never delete rows) and `content_items` identities
+ * (`id`, `url_id`) are never reassigned — only the cached `source` can
+ * change, and the sole writer of that column is the cache-aware downgrade
+ * in `resolve-content-item-id.ts` itself.
+ *
+ * NOT safe to share across processes or across two concurrent writer
+ * connections — the crawler holds a single writer connection per archive
+ * (enforced by the archive lock), so no such sharing exists.
+ */
+export interface WriteRefCaches {
+    /** `url string → url_refs.id`. */
+    readonly urlIds: Map<string, number>;
+    /** `url string → content_items` identity (id + last-known source). */
+    readonly contentItems: Map<string, ContentItemCacheEntry>;
+    /** `url string → resource_items.id`. */
+    readonly resourceIds: Map<string, number>;
+    /** `normalized content-type raw value → content_type_refs.id`. */
+    readonly contentTypeIds: Map<string, number>;
+    /** `hex(content hash) → json_refs.id`. */
+    readonly jsonIds: Map<string, number>;
+    /** `hex(content hash) → blob_refs.id`. */
+    readonly blobIds: Map<string, number>;
+    /**
+     * Header dictionary caches (`header_name_refs` / `header_value_refs` /
+     * `header_sets` id maps). `null` until the first header-set upsert
+     * warms them from the DB — warming issues three SELECTs, which a
+     * crawl that never stores headers (e.g. list-mode with no responses)
+     * should not pay for.
+     */
+    headers: HeaderTableCaches | null;
+}

package/lib/archive/db-ops/_shared/types.js

@@ -0,0 +1 @@
+export {};

package/lib/archive/db-ops/_shared/upsert-blob-ref.d.ts

@@ -0,0 +1,25 @@
+import type { WriteRefCaches } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Resolves the `blob_refs.id` for one large `data:` URI, decoding and
+ * inserting the deduplicated (hash-keyed) payload row when it is new.
+ *
+ * Callers route values here per the data-URI threshold rule (`data:` URIs
+ * longer than `DATA_URI_URL_REFS_LIMIT` — everything else belongs in
+ * `url_refs`). The payload bytes are decoded from the base64 /
+ * percent-encoded tail, hashed, and stored zstd-compressed — the same
+ * storage contract as the rows produced when migrating an existing
+ * archive (`populate-ref-tables/populate-blob-refs.ts`).
+ *
+ * Returns `null` when the data URI fails to decode (malformed base64 /
+ * truncated percent escape). A single malformed image `src` must not
+ * abort the page write; the caller stores `*_blob_id = null` and the
+ * information loss is limited to that one slot.
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ * @param caches - The connection's write-side id caches; mutated in place.
+ * @param dataUri - The raw `data:` URI value.
+ * @returns The `blob_refs.id`, or `null` when the URI cannot be decoded.
+ * @example
+ * const id = await upsertBlobRef(knex, caches, 'data:image/png;base64,<payload>');
+ */
+export declare function upsertBlobRef(qb: Knex | Knex.Transaction, caches: WriteRefCaches, dataUri: string): Promise<number | null>;