@nitpicker/archive 0.0.0 → 0.25.0

package/lib/db-ops/_shared/clear-write-ref-caches.d.ts

@@ -0,0 +1,27 @@
+import type { WriteRefCaches } from './types.js';
+/**
+ * Empties every map in a {@link WriteRefCaches} bundle in place.
+ *
+ * Called when a write transaction that used the caches rolls back. SQLite
+ * `ROLLBACK` undoes every row the transaction inserted, but `AUTOINCREMENT`
+ * counters never rewind — so an id cached optimistically during the failed
+ * attempt would survive the rollback and no longer correspond to any row.
+ * A retried attempt reusing that id would issue `UPDATE ... WHERE id = ?`
+ * against a nonexistent row and silently affect zero rows instead of
+ * failing loudly.
+ *
+ * A full clear (rather than tracking which entries the failed transaction
+ * touched) is the simplest correct fix: transient write failures are rare
+ * (retry exists precisely because they are), so paying a one-time
+ * DB-again warm-up after a rollback is cheap next to the correctness risk
+ * of a partially-poisoned cache.
+ * @param caches - The cache bundle to clear.
+ * @example
+ * try {
+ *   await knex.transaction(async (trx) => { ... });
+ * } catch (error) {
+ *   clearWriteRefCaches(caches);
+ *   throw error;
+ * }
+ */
+export declare function clearWriteRefCaches(caches: WriteRefCaches): void;

package/lib/db-ops/_shared/clear-write-ref-caches.js

@@ -0,0 +1,35 @@
+/**
+ * Empties every map in a {@link WriteRefCaches} bundle in place.
+ *
+ * Called when a write transaction that used the caches rolls back. SQLite
+ * `ROLLBACK` undoes every row the transaction inserted, but `AUTOINCREMENT`
+ * counters never rewind — so an id cached optimistically during the failed
+ * attempt would survive the rollback and no longer correspond to any row.
+ * A retried attempt reusing that id would issue `UPDATE ... WHERE id = ?`
+ * against a nonexistent row and silently affect zero rows instead of
+ * failing loudly.
+ *
+ * A full clear (rather than tracking which entries the failed transaction
+ * touched) is the simplest correct fix: transient write failures are rare
+ * (retry exists precisely because they are), so paying a one-time
+ * DB-again warm-up after a rollback is cheap next to the correctness risk
+ * of a partially-poisoned cache.
+ * @param caches - The cache bundle to clear.
+ * @example
+ * try {
+ *   await knex.transaction(async (trx) => { ... });
+ * } catch (error) {
+ *   clearWriteRefCaches(caches);
+ *   throw error;
+ * }
+ */
+export function clearWriteRefCaches(caches) {
+    caches.urlIds.clear();
+    caches.contentItems.clear();
+    caches.resourceIds.clear();
+    caches.contentTypeIds.clear();
+    caches.jsonIds.clear();
+    caches.blobIds.clear();
+    caches.consoleLogIds.clear();
+    caches.headers = null;
+}

package/lib/db-ops/_shared/compress-payload.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Compresses raw bytes for storage in a `codec`/`size_raw`/`size_stored`
+ * BLOB column, matching the read side of
+ * {@link import('./decode-json-ref.js').decodeJsonRef} and
+ * {@link import('../../decode-html-blob.js').decodeStoredBlob}.
+ *
+ * The one shared primitive behind every `codec: 'zstd'` writer in the
+ * archive (`upsert-json-ref.ts`, `write-page-html-blob.ts`,
+ * `replace-page-templates.ts`) — a codec change (e.g. a brotli fallback, a
+ * different zstd level) only has to happen here. Callers that also need a
+ * content hash (`upsert-json-ref.ts`, `write-page-html-blob.ts`) compute it
+ * themselves from the same `rawBytes` they pass in here, rather than this
+ * function computing one not every caller needs.
+ * @param rawBytes - The raw (uncompressed) payload bytes.
+ * @returns The compressed body alongside the codec and size bookkeeping the
+ *   BLOB column's schema expects.
+ * @example
+ * const rawBytes = Buffer.from(JSON.stringify(reason), 'utf8');
+ * const { body, codec, sizeRaw, sizeStored } = compressPayload(rawBytes);
+ */
+export declare function compressPayload(rawBytes: Buffer): {
+    readonly body: Buffer;
+    readonly codec: 'zstd';
+    readonly sizeRaw: number;
+    readonly sizeStored: number;
+};

package/lib/db-ops/_shared/compress-payload.js

@@ -0,0 +1,30 @@
+import { zstdCompressSync } from 'node:zlib';
+/**
+ * Compresses raw bytes for storage in a `codec`/`size_raw`/`size_stored`
+ * BLOB column, matching the read side of
+ * {@link import('./decode-json-ref.js').decodeJsonRef} and
+ * {@link import('../../decode-html-blob.js').decodeStoredBlob}.
+ *
+ * The one shared primitive behind every `codec: 'zstd'` writer in the
+ * archive (`upsert-json-ref.ts`, `write-page-html-blob.ts`,
+ * `replace-page-templates.ts`) — a codec change (e.g. a brotli fallback, a
+ * different zstd level) only has to happen here. Callers that also need a
+ * content hash (`upsert-json-ref.ts`, `write-page-html-blob.ts`) compute it
+ * themselves from the same `rawBytes` they pass in here, rather than this
+ * function computing one not every caller needs.
+ * @param rawBytes - The raw (uncompressed) payload bytes.
+ * @returns The compressed body alongside the codec and size bookkeeping the
+ *   BLOB column's schema expects.
+ * @example
+ * const rawBytes = Buffer.from(JSON.stringify(reason), 'utf8');
+ * const { body, codec, sizeRaw, sizeStored } = compressPayload(rawBytes);
+ */
+export function compressPayload(rawBytes) {
+    const compressed = zstdCompressSync(rawBytes);
+    return {
+        body: compressed,
+        codec: 'zstd',
+        sizeRaw: rawBytes.byteLength,
+        sizeStored: compressed.byteLength,
+    };
+}

package/lib/db-ops/_shared/create-write-ref-caches.d.ts

@@ -0,0 +1,17 @@
+import type { WriteRefCaches } from './types.js';
+/**
+ * Creates an empty {@link WriteRefCaches} bundle for one archive writer
+ * connection.
+ *
+ * All maps start empty and fill lazily as the write path touches URLs,
+ * content types, JSON payloads, and data-URI blobs. The header caches
+ * (`headers`) start as `null` and are warmed from the DB by the first
+ * header-set upsert — see {@link ./types.ts} for the cache-correctness
+ * argument (append-only ref tables + single writer connection).
+ * @returns A fresh cache bundle. One per `Database` instance; never share
+ *   across connections.
+ * @example
+ * const caches = createWriteRefCaches();
+ * const urlId = await upsertUrlRef(knex, caches, 'https://example.com/');
+ */
+export declare function createWriteRefCaches(): WriteRefCaches;

package/lib/db-ops/_shared/create-write-ref-caches.js

@@ -0,0 +1,27 @@
+/**
+ * Creates an empty {@link WriteRefCaches} bundle for one archive writer
+ * connection.
+ *
+ * All maps start empty and fill lazily as the write path touches URLs,
+ * content types, JSON payloads, and data-URI blobs. The header caches
+ * (`headers`) start as `null` and are warmed from the DB by the first
+ * header-set upsert — see {@link ./types.ts} for the cache-correctness
+ * argument (append-only ref tables + single writer connection).
+ * @returns A fresh cache bundle. One per `Database` instance; never share
+ *   across connections.
+ * @example
+ * const caches = createWriteRefCaches();
+ * const urlId = await upsertUrlRef(knex, caches, 'https://example.com/');
+ */
+export function createWriteRefCaches() {
+    return {
+        urlIds: new Map(),
+        contentItems: new Map(),
+        resourceIds: new Map(),
+        contentTypeIds: new Map(),
+        jsonIds: new Map(),
+        blobIds: new Map(),
+        consoleLogIds: new Map(),
+        headers: null,
+    };
+}

package/lib/db-ops/_shared/decode-json-ref.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Decodes one `json_refs` payload back into its JSON string.
+ *
+ * The write path (`upsert-json-ref.ts`) and the archive-migration
+ * populate (`populate-json-refs.ts`) both store bodies zstd-compressed;
+ * the `'none'` codec is accepted because the `json_refs.codec` column
+ * permits it, not because either writer currently produces it. Corrupt
+ * bodies fail closed to `null` rather than throwing — readers treat an
+ * undecodable payload the same as an absent one.
+ * @param body - The raw `json_refs.json_text` body, or null when absent.
+ * @param codec - The `json_refs.codec` value, or null when absent.
+ * @returns The decoded JSON string, or null when there is no body or the
+ *   body cannot be decoded.
+ * @example
+ * const metaExtras = decodeJsonRef(row.extras_body, row.extras_codec);
+ */
+export declare function decodeJsonRef(body: Buffer | string | null, codec: 'zstd' | 'none' | null): string | null;

package/lib/db-ops/_shared/decode-json-ref.js

@@ -0,0 +1,31 @@
+import { zstdDecompressSync } from 'node:zlib';
+/**
+ * Decodes one `json_refs` payload back into its JSON string.
+ *
+ * The write path (`upsert-json-ref.ts`) and the archive-migration
+ * populate (`populate-json-refs.ts`) both store bodies zstd-compressed;
+ * the `'none'` codec is accepted because the `json_refs.codec` column
+ * permits it, not because either writer currently produces it. Corrupt
+ * bodies fail closed to `null` rather than throwing — readers treat an
+ * undecodable payload the same as an absent one.
+ * @param body - The raw `json_refs.json_text` body, or null when absent.
+ * @param codec - The `json_refs.codec` value, or null when absent.
+ * @returns The decoded JSON string, or null when there is no body or the
+ *   body cannot be decoded.
+ * @example
+ * const metaExtras = decodeJsonRef(row.extras_body, row.extras_codec);
+ */
+export function decodeJsonRef(body, codec) {
+    if (body == null) {
+        return null;
+    }
+    try {
+        if (codec === 'zstd') {
+            return zstdDecompressSync(body).toString('utf8');
+        }
+        return typeof body === 'string' ? body : body.toString('utf8');
+    }
+    catch {
+        return null;
+    }
+}

package/lib/db-ops/_shared/keyset-paginate-by-id.d.ts

@@ -0,0 +1,49 @@
+import type { Knex } from 'knex';
+/**
+ * Reads every row a caller-supplied query matches, in ascending-`id`
+ * keyset-paginated chunks rather than one unbounded SELECT (issue #294): on
+ * a large archive, a single multi-second-to-minutes scan gives the caller
+ * no way to report progress mid-read. The accumulated result is identical
+ * to a single un-chunked SELECT — chunking exists purely to make the scan
+ * observable, not to bound memory (the mapped output is already fully
+ * materialised for the caller either way).
+ *
+ * Extracted from `getResourceUrlList` / `listDedupeCapObservations`, which
+ * had grown byte-for-byte identical pagination boilerplate (the `MAX(id)`
+ * probe, the `for (;;)` chunk loop, the `onProgress` shape) around two
+ * otherwise-unrelated queries. Callers own everything query-shaped
+ * (joins, `WHERE`, `SELECT`, chunk size via `.limit()` inside
+ * `buildQuery`) — this helper only owns the keyset-chunking mechanics
+ * common to both.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param idTable - The table `id` keys off, used for the O(1) `MAX(id)`
+ *   probe that seeds `onProgress`'s total (only queried when `onProgress`
+ *   is given).
+ * @param buildQuery - Builds one chunk's query given the last-seen `id`
+ *   (`0` on the first call). Must filter on `id > lastId`, order by `id`
+ *   ascending, and cap the row count (typically via `.limit()`) — this
+ *   helper does not add any of those clauses itself. The resolved rows
+ *   must each carry an `id` field so the next chunk's cursor can advance.
+ * @param mapRow - Transforms one raw row into the caller's output shape.
+ * @param onProgress - Called after each chunk, with the highest `id`
+ *   scanned so far and the max `id` in `idTable`. Omit for no reporting.
+ * @returns Every matched row, mapped via `mapRow`, in `id` order.
+ * @example
+ * ```ts
+ * const urls = await keysetPaginateById(
+ * 	knex,
+ * 	'resource_items',
+ * 	(lastId) =>
+ * 		knex('resource_items')
+ * 			.join('url_refs', 'url_refs.id', 'resource_items.url_id')
+ * 			.where('resource_items.id', '>', lastId)
+ * 			.orderBy('resource_items.id', 'asc')
+ * 			.limit(2000)
+ * 			.select('resource_items.id as id', 'url_refs.url as url'),
+ * 	(row) => row.url,
+ * );
+ * ```
+ */
+export declare function keysetPaginateById<Row extends {
+    id: number;
+}, Out>(knex: Knex, idTable: string, buildQuery: (lastId: number) => Knex.QueryBuilder, mapRow: (row: Row) => Out, onProgress?: (scannedUpToId: number, maxId: number) => void): Promise<Out[]>;

package/lib/db-ops/_shared/keyset-paginate-by-id.js

@@ -0,0 +1,69 @@
+/**
+ * Reads every row a caller-supplied query matches, in ascending-`id`
+ * keyset-paginated chunks rather than one unbounded SELECT (issue #294): on
+ * a large archive, a single multi-second-to-minutes scan gives the caller
+ * no way to report progress mid-read. The accumulated result is identical
+ * to a single un-chunked SELECT — chunking exists purely to make the scan
+ * observable, not to bound memory (the mapped output is already fully
+ * materialised for the caller either way).
+ *
+ * Extracted from `getResourceUrlList` / `listDedupeCapObservations`, which
+ * had grown byte-for-byte identical pagination boilerplate (the `MAX(id)`
+ * probe, the `for (;;)` chunk loop, the `onProgress` shape) around two
+ * otherwise-unrelated queries. Callers own everything query-shaped
+ * (joins, `WHERE`, `SELECT`, chunk size via `.limit()` inside
+ * `buildQuery`) — this helper only owns the keyset-chunking mechanics
+ * common to both.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param idTable - The table `id` keys off, used for the O(1) `MAX(id)`
+ *   probe that seeds `onProgress`'s total (only queried when `onProgress`
+ *   is given).
+ * @param buildQuery - Builds one chunk's query given the last-seen `id`
+ *   (`0` on the first call). Must filter on `id > lastId`, order by `id`
+ *   ascending, and cap the row count (typically via `.limit()`) — this
+ *   helper does not add any of those clauses itself. The resolved rows
+ *   must each carry an `id` field so the next chunk's cursor can advance.
+ * @param mapRow - Transforms one raw row into the caller's output shape.
+ * @param onProgress - Called after each chunk, with the highest `id`
+ *   scanned so far and the max `id` in `idTable`. Omit for no reporting.
+ * @returns Every matched row, mapped via `mapRow`, in `id` order.
+ * @example
+ * ```ts
+ * const urls = await keysetPaginateById(
+ * 	knex,
+ * 	'resource_items',
+ * 	(lastId) =>
+ * 		knex('resource_items')
+ * 			.join('url_refs', 'url_refs.id', 'resource_items.url_id')
+ * 			.where('resource_items.id', '>', lastId)
+ * 			.orderBy('resource_items.id', 'asc')
+ * 			.limit(2000)
+ * 			.select('resource_items.id as id', 'url_refs.url as url'),
+ * 	(row) => row.url,
+ * );
+ * ```
+ */
+export async function keysetPaginateById(knex, idTable, buildQuery, mapRow, onProgress) {
+    // MAX() over the keyset column is an O(1) index-tail read; only fetched
+    // when someone is listening.
+    let maxId = 0;
+    if (onProgress) {
+        const [maxRow] = await knex(idTable).max({ max: 'id' });
+        maxId = maxRow?.max ?? 0;
+    }
+    const results = [];
+    let lastId = 0;
+    for (;;) {
+        const rows = (await buildQuery(lastId));
+        if (rows.length === 0) {
+            onProgress?.(maxId, maxId);
+            break;
+        }
+        lastId = rows.at(-1).id;
+        for (const row of rows) {
+            results.push(mapRow(row));
+        }
+        onProgress?.(Math.min(lastId, maxId), maxId);
+    }
+    return results;
+}

package/lib/db-ops/_shared/load-response-headers-by-set-ids.d.ts

@@ -0,0 +1,20 @@
+import type { Knex } from 'knex';
+/**
+ * Loads and merges the response headers for every given `header_sets.id`
+ * back into flat `Record<name, value>` objects, in one chunked batch
+ * (never one query per row).
+ *
+ * Multi-value headers (repeated `Set-Cookie` etc.) are joined with
+ * `', '` in `occurrence` order within each name — the flat-record shape
+ * the pre-0.13 `responseHeaders` JSON column stored, which every reader
+ * of the reconstructed rows still expects.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param headerSetIds - Distinct `header_sets.id` values to load. Empty
+ *   input returns an empty map without touching the DB.
+ * @returns Map from `header_sets.id` to the merged header record. Ids
+ *   with no entries are absent from the map.
+ * @example
+ * const headersBySetId = await loadResponseHeadersBySetIds(knex, [1, 2]);
+ * const record = headersBySetId.get(1) ?? {};
+ */
+export declare function loadResponseHeadersBySetIds(knex: Knex, headerSetIds: readonly number[]): Promise<Map<number, Record<string, string>>>;

package/lib/db-ops/_shared/load-response-headers-by-set-ids.js

@@ -0,0 +1,53 @@
+/**
+ * Ids per `WHERE header_set_id IN (?, ...)` chunk. Modern SQLite builds
+ * (including the vendored libsql) reject statements binding more than
+ * 32,766 parameters, and header sets dedupe poorly on real sites (Date /
+ * ETag / Set-Cookie make most sets unique), so an IN-list sized by the
+ * page batch without chunking would exceed the limit on large archives.
+ * 800 matches
+ * the `LOOKUP_CHUNK_SIZE` convention the populate/upsert paths already
+ * use (`resolve-text-refs.ts` and siblings).
+ */
+const LOOKUP_CHUNK_SIZE = 800;
+/**
+ * Loads and merges the response headers for every given `header_sets.id`
+ * back into flat `Record<name, value>` objects, in one chunked batch
+ * (never one query per row).
+ *
+ * Multi-value headers (repeated `Set-Cookie` etc.) are joined with
+ * `', '` in `occurrence` order within each name — the flat-record shape
+ * the pre-0.13 `responseHeaders` JSON column stored, which every reader
+ * of the reconstructed rows still expects.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param headerSetIds - Distinct `header_sets.id` values to load. Empty
+ *   input returns an empty map without touching the DB.
+ * @returns Map from `header_sets.id` to the merged header record. Ids
+ *   with no entries are absent from the map.
+ * @example
+ * const headersBySetId = await loadResponseHeadersBySetIds(knex, [1, 2]);
+ * const record = headersBySetId.get(1) ?? {};
+ */
+export async function loadResponseHeadersBySetIds(knex, headerSetIds) {
+    const merged = new Map();
+    for (let index = 0; index < headerSetIds.length; index += LOOKUP_CHUNK_SIZE) {
+        const chunk = headerSetIds.slice(index, index + LOOKUP_CHUNK_SIZE);
+        const headerRows = (await knex('header_set_entries as hse')
+            .join('header_name_refs as hnr', 'hnr.id', 'hse.name_id')
+            .join('header_value_refs as hvr', 'hvr.id', 'hse.value_id')
+            .whereIn('hse.header_set_id', chunk)
+            .orderBy(['hse.header_set_id', 'hnr.name', 'hse.occurrence'])
+            .select('hse.header_set_id as headerSetId', 'hnr.name as name', 'hvr.value as value'));
+        for (const row of headerRows) {
+            const bySet = merged.get(row.headerSetId) ?? new Map();
+            const values = bySet.get(row.name) ?? [];
+            values.push(row.value);
+            bySet.set(row.name, values);
+            merged.set(row.headerSetId, bySet);
+        }
+    }
+    const result = new Map();
+    for (const [setId, bySet] of merged) {
+        result.set(setId, Object.fromEntries([...bySet.entries()].map(([k, v]) => [k, v.join(', ')])));
+    }
+    return result;
+}

package/lib/db-ops/_shared/resolve-content-item-id.d.ts

@@ -0,0 +1,62 @@
+import type { ResolveContentItemIdOptions, WriteRefCaches } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Returns the `content_items.id` for `url`, inserting a placeholder row
+ * (via `url_refs`) when the URL is not yet known. Foundational identity
+ * primitive shared by every op that needs to record an edge or lookup to
+ * a URL — the `content_items` successor of the legacy `pages.id`
+ * resolver.
+ *
+ * **Source semantics.** `source` is written ONLY on the INSERT path —
+ * when the row already exists, the INSERT is never reached and the
+ * existing row's `source` stays untouched. This is what keeps a second
+ * `crawl --inventory` from "demoting" a page that was first labelled
+ * `'inventory-seed'` back to `'inventory-discovered'` on later passes.
+ *
+ * **Crawled-wins downgrade.** When a row that was previously labelled
+ * `'inventory-seed'` or `'inventory-discovered'` is re-encountered via a
+ * `'crawled'`-lineage anchor (the parent page is part of the graph
+ * reachable from the original crawl roots), the row is downgraded to
+ * `'crawled'`. The inventory goal is finding orphans — anything reachable
+ * from the crawled chain is NOT an orphan and should not retain an
+ * inventory label. The cached `source` is updated in the same step so a
+ * later hit does not re-issue the UPDATE.
+ *
+ * **`is_metadata_only` promotion** (issue #369) is evaluated in the same
+ * pass as the crawled-wins downgrade, and both diffs are folded into a
+ * single `UPDATE` statement (not two) when either column actually
+ * changes — see {@link applyExistingRowUpdates}.
+ *
+ * **Cache poisoning on rollback.** Entries cached inside a transaction
+ * that later rolls back would point at ids that no longer exist. Every
+ * write op that opens a multi-statement transaction around this function
+ * must discard the whole cache bundle when that transaction fails,
+ * before any retry — see the try/catch + `clearWriteRefCaches` wrappers
+ * in `update-page.ts` and `record-redirect.ts`.
+ *
+ * **Race-condition safety.** The INSERT uses `ON CONFLICT(url_id) DO
+ * UPDATE SET url_id = url_id RETURNING id, source` — the same no-op
+ * -update idiom as `upsertUrlRef` — so when a concurrent transaction
+ * wins the insert race, the statement still returns the existing row's
+ * id and source in one round trip. `onConflict().ignore()` without
+ * `RETURNING` must NOT be used here: knex's sqlite-family dialects
+ * report the connection's stale `lastInsertRowid` for a conflict-ignored
+ * insert, which reads as a valid id belonging to an unrelated row.
+ * @param qb - Knex instance OR a transaction, used verbatim for every
+ *   statement this function issues. Pass the `trx` when running inside a
+ *   transaction so reads see uncommitted writes from that transaction.
+ * @param caches - The connection's write-side id caches; mutated in place.
+ * @param url - The URL to look up or insert (normalised
+ *   `withoutHashAndAuth` form, matching the legacy identity contract).
+ * @param options - See {@link ResolveContentItemIdOptions}.
+ * @returns The `content_items.id` of the existing or newly inserted row.
+ * @throws {Error} When the upsert's `RETURNING` yields no row — should
+ *   not happen, so it surfaces as a hard error.
+ * @example
+ * const pageId = await resolveContentItemId(trx, caches, anchor.href, {
+ *   isExternal: 1,
+ *   source: 'crawled',
+ *   isMetadataOnly: 1,
+ * });
+ */
+export declare function resolveContentItemId(qb: Knex | Knex.Transaction, caches: WriteRefCaches, url: string, options?: ResolveContentItemIdOptions): Promise<number>;

package/lib/db-ops/_shared/resolve-content-item-id.js

@@ -0,0 +1,143 @@
+import { upsertUrlRef } from './upsert-url-ref.js';
+/**
+ * Returns the `content_items.id` for `url`, inserting a placeholder row
+ * (via `url_refs`) when the URL is not yet known. Foundational identity
+ * primitive shared by every op that needs to record an edge or lookup to
+ * a URL — the `content_items` successor of the legacy `pages.id`
+ * resolver.
+ *
+ * **Source semantics.** `source` is written ONLY on the INSERT path —
+ * when the row already exists, the INSERT is never reached and the
+ * existing row's `source` stays untouched. This is what keeps a second
+ * `crawl --inventory` from "demoting" a page that was first labelled
+ * `'inventory-seed'` back to `'inventory-discovered'` on later passes.
+ *
+ * **Crawled-wins downgrade.** When a row that was previously labelled
+ * `'inventory-seed'` or `'inventory-discovered'` is re-encountered via a
+ * `'crawled'`-lineage anchor (the parent page is part of the graph
+ * reachable from the original crawl roots), the row is downgraded to
+ * `'crawled'`. The inventory goal is finding orphans — anything reachable
+ * from the crawled chain is NOT an orphan and should not retain an
+ * inventory label. The cached `source` is updated in the same step so a
+ * later hit does not re-issue the UPDATE.
+ *
+ * **`is_metadata_only` promotion** (issue #369) is evaluated in the same
+ * pass as the crawled-wins downgrade, and both diffs are folded into a
+ * single `UPDATE` statement (not two) when either column actually
+ * changes — see {@link applyExistingRowUpdates}.
+ *
+ * **Cache poisoning on rollback.** Entries cached inside a transaction
+ * that later rolls back would point at ids that no longer exist. Every
+ * write op that opens a multi-statement transaction around this function
+ * must discard the whole cache bundle when that transaction fails,
+ * before any retry — see the try/catch + `clearWriteRefCaches` wrappers
+ * in `update-page.ts` and `record-redirect.ts`.
+ *
+ * **Race-condition safety.** The INSERT uses `ON CONFLICT(url_id) DO
+ * UPDATE SET url_id = url_id RETURNING id, source` — the same no-op
+ * -update idiom as `upsertUrlRef` — so when a concurrent transaction
+ * wins the insert race, the statement still returns the existing row's
+ * id and source in one round trip. `onConflict().ignore()` without
+ * `RETURNING` must NOT be used here: knex's sqlite-family dialects
+ * report the connection's stale `lastInsertRowid` for a conflict-ignored
+ * insert, which reads as a valid id belonging to an unrelated row.
+ * @param qb - Knex instance OR a transaction, used verbatim for every
+ *   statement this function issues. Pass the `trx` when running inside a
+ *   transaction so reads see uncommitted writes from that transaction.
+ * @param caches - The connection's write-side id caches; mutated in place.
+ * @param url - The URL to look up or insert (normalised
+ *   `withoutHashAndAuth` form, matching the legacy identity contract).
+ * @param options - See {@link ResolveContentItemIdOptions}.
+ * @returns The `content_items.id` of the existing or newly inserted row.
+ * @throws {Error} When the upsert's `RETURNING` yields no row — should
+ *   not happen, so it surfaces as a hard error.
+ * @example
+ * const pageId = await resolveContentItemId(trx, caches, anchor.href, {
+ *   isExternal: 1,
+ *   source: 'crawled',
+ *   isMetadataOnly: 1,
+ * });
+ */
+export async function resolveContentItemId(qb, caches, url, options) {
+    const { isExternal, source, isMetadataOnly } = options ?? {};
+    const cached = caches.contentItems.get(url);
+    if (cached !== undefined) {
+        await applyExistingRowUpdates(qb, cached, source, isMetadataOnly);
+        return cached.id;
+    }
+    const urlId = await upsertUrlRef(qb, caches, url);
+    const [record] = (await qb
+        .select('id', 'source', 'is_metadata_only')
+        .from('content_items')
+        .where('url_id', urlId));
+    if (record !== undefined) {
+        const entry = {
+            id: record.id,
+            source: record.source,
+            isMetadataOnly: record.is_metadata_only,
+        };
+        await applyExistingRowUpdates(qb, entry, source, isMetadataOnly);
+        caches.contentItems.set(url, entry);
+        return entry.id;
+    }
+    const insertedRows = await qb.raw(`INSERT INTO content_items (url_id, scraped, is_target, is_external, is_metadata_only${source === undefined ? '' : ', source'})
+			 VALUES (?, 0, 0, ?, ?${source === undefined ? '' : ', ?'})
+			 ON CONFLICT(url_id) DO UPDATE SET url_id = url_id
+			 RETURNING id, source, is_metadata_only`, source === undefined
+        ? [urlId, isExternal ?? 0, isMetadataOnly ?? 0]
+        : [urlId, isExternal ?? 0, isMetadataOnly ?? 0, source]);
+    const inserted = insertedRows[0];
+    if (inserted === undefined) {
+        throw new Error(`Failed to insert a new content item: ${url}`);
+    }
+    const insertedEntry = {
+        id: inserted.id,
+        source: inserted.source,
+        isMetadataOnly: inserted.is_metadata_only,
+    };
+    // A conflict means a concurrent writer created the row between this
+    // function's SELECT miss and the INSERT — the returned `source` /
+    // `is_metadata_only` are that row's values, so both follow-ups must be
+    // evaluated exactly as on the SELECT-hit path.
+    await applyExistingRowUpdates(qb, insertedEntry, source, isMetadataOnly);
+    caches.contentItems.set(url, insertedEntry);
+    return insertedEntry.id;
+}
+/**
+ * Applies both the crawled-wins `source` downgrade and the `is_metadata_only`
+ * promotion (issue #369) to an already-resolved row in a single `UPDATE`,
+ * and keeps the cache entry in sync so a repeat call with the same values
+ * does not re-issue any write.
+ *
+ * Folded into one function (and one statement) rather than two independent
+ * ones: `resolveContentItemId` runs once per anchor on every scraped page
+ * across a whole crawl, so a page whose anchors trip both conditions would
+ * otherwise pay two round trips instead of one.
+ * @param qb - Knex instance or transaction.
+ * @param entry - The cached identity to check and mutate.
+ * @param source - The resolution's lineage label, if any — see
+ *   {@link ResolveContentItemIdOptions.source}.
+ * @param isMetadataOnly - The resolution's explicit scrape-depth opinion, if
+ *   any — see {@link ResolveContentItemIdOptions.isMetadataOnly}. Omitted by
+ *   every caller except `replaceAnchorEdges`, which always has an opinion,
+ *   so this is a no-op for the rest.
+ */
+async function applyExistingRowUpdates(qb, entry, source, isMetadataOnly) {
+    const updates = {};
+    if (source === 'crawled' && entry.source !== 'crawled') {
+        updates.source = 'crawled';
+    }
+    if (isMetadataOnly !== undefined && entry.isMetadataOnly !== isMetadataOnly) {
+        updates.is_metadata_only = isMetadataOnly;
+    }
+    if (Object.keys(updates).length === 0) {
+        return;
+    }
+    await qb('content_items').where('id', entry.id).update(updates);
+    if (updates.source !== undefined) {
+        entry.source = updates.source;
+    }
+    if (updates.is_metadata_only !== undefined) {
+        entry.isMetadataOnly = updates.is_metadata_only;
+    }
+}

package/lib/db-ops/_shared/resolve-url-or-blob.d.ts

@@ -0,0 +1,23 @@
+import type { WriteRefCaches } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Routes one URL-shaped value to either `url_refs` or `blob_refs` per the
+ * data-URI threshold rule (large data URIs live in `blob_refs`, everything
+ * else in `url_refs`), upserting into whichever dictionary applies.
+ *
+ * Shared by every writer that can receive a `data:` URI in a URL-shaped
+ * column: `image_items.src` / `current_src` and `resource_items`' own
+ * identity URL. At most one of `url` / `blob` is non-null; both are `null`
+ * when the value is empty or a malformed data URI that fails to decode.
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param value - Raw URL-shaped value.
+ * @returns `{ url, blob }` pair with at most one non-null field.
+ * @example
+ * const slot = await resolveUrlOrBlob(trx, caches, image.src);
+ * // slot.url set for a regular URL, slot.blob set for a large data: URI.
+ */
+export declare function resolveUrlOrBlob(qb: Knex | Knex.Transaction, caches: WriteRefCaches, value: string | null | undefined): Promise<{
+    url: number | null;
+    blob: number | null;
+}>;