@nitpicker/archive 0.0.0 → 0.25.1

package/lib/db-ops/dedupe-cap/accumulate-dedupe-cap-rejected-count.d.ts

@@ -0,0 +1,18 @@
+import type { Knex } from 'knex';
+/**
+ * Adds `rejectedCount` onto the `rejected_count` of the `dedupe_cap_events`
+ * row for `shapeKey`, treating a still-`NULL` count as `0`. Unlike
+ * `finalize-dedupe-cap-event.ts` (which stamps a session's own newly-capped
+ * shape exactly once, guarded by `whereNull`), this targets a shape that
+ * capped in an EARLIER session and was preloaded into `DedupeCapTracker`'s
+ * sticky set (see `DedupeCapTracker`'s constructor JSDoc) — gate rejections
+ * for such a shape still occur in the current session, but no `dedupeCap`
+ * event (and thus no new row) is ever emitted for it, since the tracker
+ * short-circuits on an already-sticky shape before `observe` runs. Matches
+ * by `shape_key` rather than `id` because the caller (`CrawlerOrchestrator`)
+ * only has the shape key for a preloaded-sticky shape, never its row id.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param shapeKey - The capped shape whose rejection count to accumulate.
+ * @param rejectedCount - Additional anchors rejected for this shape in the current session.
+ */
+export declare function accumulateDedupeCapRejectedCount(knex: Knex, shapeKey: string, rejectedCount: number): Promise<void>;

package/lib/db-ops/dedupe-cap/accumulate-dedupe-cap-rejected-count.js

@@ -0,0 +1,23 @@
+/**
+ * Adds `rejectedCount` onto the `rejected_count` of the `dedupe_cap_events`
+ * row for `shapeKey`, treating a still-`NULL` count as `0`. Unlike
+ * `finalize-dedupe-cap-event.ts` (which stamps a session's own newly-capped
+ * shape exactly once, guarded by `whereNull`), this targets a shape that
+ * capped in an EARLIER session and was preloaded into `DedupeCapTracker`'s
+ * sticky set (see `DedupeCapTracker`'s constructor JSDoc) — gate rejections
+ * for such a shape still occur in the current session, but no `dedupeCap`
+ * event (and thus no new row) is ever emitted for it, since the tracker
+ * short-circuits on an already-sticky shape before `observe` runs. Matches
+ * by `shape_key` rather than `id` because the caller (`CrawlerOrchestrator`)
+ * only has the shape key for a preloaded-sticky shape, never its row id.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param shapeKey - The capped shape whose rejection count to accumulate.
+ * @param rejectedCount - Additional anchors rejected for this shape in the current session.
+ */
+export async function accumulateDedupeCapRejectedCount(knex, shapeKey, rejectedCount) {
+    await knex('dedupe_cap_events')
+        .where({ shape_key: shapeKey })
+        .update({
+        rejected_count: knex.raw('COALESCE(rejected_count, 0) + ?', [rejectedCount]),
+    });
+}

package/lib/db-ops/dedupe-cap/finalize-dedupe-cap-event.d.ts

@@ -0,0 +1,12 @@
+import type { Knex } from 'knex';
+/**
+ * Finalizes a `dedupe_cap_events` row by stamping `rejected_count` — but
+ * ONLY if it is still unset. The `whereNull('rejected_count')` guard makes
+ * this idempotent, mirroring `close-network-outage.ts`'s `ended_at` guard: a
+ * second call matches zero rows and is a silent no-op rather than
+ * overwriting an already-finalized count.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param id - The `dedupe_cap_events.id` to finalize.
+ * @param rejectedCount - Number of anchors rejected for this shape after it capped.
+ */
+export declare function finalizeDedupeCapEvent(knex: Knex, id: number, rejectedCount: number): Promise<void>;

package/lib/db-ops/dedupe-cap/finalize-dedupe-cap-event.js

@@ -0,0 +1,15 @@
+/**
+ * Finalizes a `dedupe_cap_events` row by stamping `rejected_count` — but
+ * ONLY if it is still unset. The `whereNull('rejected_count')` guard makes
+ * this idempotent, mirroring `close-network-outage.ts`'s `ended_at` guard: a
+ * second call matches zero rows and is a silent no-op rather than
+ * overwriting an already-finalized count.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param id - The `dedupe_cap_events.id` to finalize.
+ * @param rejectedCount - Number of anchors rejected for this shape after it capped.
+ */
+export async function finalizeDedupeCapEvent(knex, id, rejectedCount) {
+    await knex('dedupe_cap_events').where({ id }).whereNull('rejected_count').update({
+        rejected_count: rejectedCount,
+    });
+}

package/lib/db-ops/dedupe-cap/insert-dedupe-cap-event.d.ts

@@ -0,0 +1,14 @@
+import type { InsertDedupeCapEventParams } from '../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Appends one row to the `dedupe_cap_events` journal, with `rejected_count`
+ * left `NULL` — the row starts life without a finalized rejection count.
+ *
+ * Called the instant `DedupeCapTracker#observe` confirms a URL shape as a
+ * same-cluster trap (the `dedupeCap` event). See
+ * `finalize-dedupe-cap-event.ts` for how `rejected_count` is later set.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param params - The newly-capped shape's fields to record.
+ * @returns The autoincremented `id` of the newly-inserted row.
+ */
+export declare function insertDedupeCapEvent(knex: Knex, params: InsertDedupeCapEventParams): Promise<number>;

package/lib/db-ops/dedupe-cap/insert-dedupe-cap-event.js

@@ -0,0 +1,30 @@
+/**
+ * Appends one row to the `dedupe_cap_events` journal, with `rejected_count`
+ * left `NULL` — the row starts life without a finalized rejection count.
+ *
+ * Called the instant `DedupeCapTracker#observe` confirms a URL shape as a
+ * same-cluster trap (the `dedupeCap` event). See
+ * `finalize-dedupe-cap-event.ts` for how `rejected_count` is later set.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param params - The newly-capped shape's fields to record.
+ * @returns The autoincremented `id` of the newly-inserted row.
+ */
+export async function insertDedupeCapEvent(knex, params) {
+    const inserted = await knex
+        .from('dedupe_cap_events')
+        .insert({
+        shape_key: params.shapeKey,
+        sample_url: params.sampleUrl,
+        body_hash: params.bodyHash,
+        effective_threshold: params.effectiveThreshold,
+        observed_count: params.observedCount,
+        detected_at: params.detectedAt,
+        rejected_count: null,
+    })
+        .returning('id');
+    const id = inserted[0]?.id;
+    if (typeof id !== 'number') {
+        throw new TypeError('insertDedupeCapEvent: INSERT returned no row id');
+    }
+    return id;
+}

package/lib/db-ops/dedupe-cap/list-dedupe-cap-observations.d.ts

@@ -0,0 +1,57 @@
+import type { DedupeCapObservationRow } from '../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Reads back every previously-scraped internal page's raw fields in the
+ * exact population a live crawl's `DedupeCapTracker#observe` call site
+ * would have fed it, so `CrawlerOrchestrator`'s five resuming-session
+ * methods (`append`/`inventory`/`recrawl`/`retryFailed`/`resume`) can
+ * replay a prior session's Misra-Gries observations into a fresh tracker
+ * instance (see `buildDedupeCapObservation`) instead of restarting every
+ * not-yet-capped shape's counter at 0.
+ *
+ * The WHERE clause reproduces the live observation gate
+ * (`!isExternal && !isMetadataOnly && html.length > 0`) using columns that
+ * survive a process restart:
+ *
+ * - `is_external = 0` — external pages carry no useful signal (same
+ *   exclusion `computeMetaSignature`'s design already assumes).
+ * - `is_target = 1` — the archived equivalent of "not metadata-only": a
+ *   metadata-only fetch never invokes the browser and is written with
+ *   `is_target = 0` (see `Crawler`'s metadata-only branch), matching the
+ *   live gate's `!isMetadataOnly` exactly.
+ * - `redirect_dest_id IS NULL` — a redirect source's own body was never
+ *   rendered; only the destination page (a separate row) was.
+ * - `is_skipped IS NULL OR is_skipped = 0` — a skipped placeholder row
+ *   (`source = 'inventory-seed'` etc.) has no rendered body either.
+ * - `pm.body_hash IS NOT NULL` — the archived proxy for `html.length > 0`:
+ *   `update-page.ts` only ever writes a non-null `body_hash` on the same
+ *   `writeHtml && html.length > 0` gate the live site's `precomputedBodyHash`
+ *   computation uses, so this column faithfully reconstructs that
+ *   condition. Also excludes pre-`body_hash`-migration legacy archives'
+ *   un-backfilled rows (`migrate-page-meta-body-hash.ts` adds the column on
+ *   every writer open but does not compute values for existing rows) —
+ *   understating the replayed count is safe (never over-counts), so no
+ *   column-presence guard is needed here.
+ *
+ * Deliberately does NOT exclude rows already marked
+ * `content_items.dedupe_cap_event_id` (the post-hoc marking column) —
+ * unlike `body_hash`, that column is only backfilled during a
+ * `viewer-build`, not written during a live crawl, so filtering on it here
+ * would silently diverge from what the live crawl-time gate actually saw.
+ * `DedupeCapTracker#observe` is an O(1) no-op for a shape already in its
+ * sticky set regardless, so replaying an already-capped shape's rows costs
+ * a discarded observation, never an incorrect one.
+ *
+ * Rows are read in `ci.id` order (insertion / discovery order) via
+ * `keysetPaginateById` — Misra-Gries counting is order-dependent, so
+ * replaying in the archive's original discovery order reproduces what the
+ * tracker's state would look like had the process never restarted, subject
+ * to the same `mapCap` LRU eviction.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param onProgress - Called after each chunk, with the highest `ci.id`
+ *   scanned so far and the max `ci.id` in the table — mirrors
+ *   `getResourceUrlList`'s progress shape (`SetupProgressCallbacks.onChunkProgress`).
+ *   Omit for no reporting (tests, and callers that don't need it).
+ * @returns Every qualifying page's raw fields, in `ci.id` order.
+ */
+export declare function listDedupeCapObservations(knex: Knex, onProgress?: (scannedUpToId: number, maxId: number) => void): Promise<DedupeCapObservationRow[]>;

package/lib/db-ops/dedupe-cap/list-dedupe-cap-observations.js

@@ -0,0 +1,87 @@
+import { keysetPaginateById } from '../_shared/keyset-paginate-by-id.js';
+const READ_CHUNK_SIZE = 2000;
+/**
+ * Reads back every previously-scraped internal page's raw fields in the
+ * exact population a live crawl's `DedupeCapTracker#observe` call site
+ * would have fed it, so `CrawlerOrchestrator`'s five resuming-session
+ * methods (`append`/`inventory`/`recrawl`/`retryFailed`/`resume`) can
+ * replay a prior session's Misra-Gries observations into a fresh tracker
+ * instance (see `buildDedupeCapObservation`) instead of restarting every
+ * not-yet-capped shape's counter at 0.
+ *
+ * The WHERE clause reproduces the live observation gate
+ * (`!isExternal && !isMetadataOnly && html.length > 0`) using columns that
+ * survive a process restart:
+ *
+ * - `is_external = 0` — external pages carry no useful signal (same
+ *   exclusion `computeMetaSignature`'s design already assumes).
+ * - `is_target = 1` — the archived equivalent of "not metadata-only": a
+ *   metadata-only fetch never invokes the browser and is written with
+ *   `is_target = 0` (see `Crawler`'s metadata-only branch), matching the
+ *   live gate's `!isMetadataOnly` exactly.
+ * - `redirect_dest_id IS NULL` — a redirect source's own body was never
+ *   rendered; only the destination page (a separate row) was.
+ * - `is_skipped IS NULL OR is_skipped = 0` — a skipped placeholder row
+ *   (`source = 'inventory-seed'` etc.) has no rendered body either.
+ * - `pm.body_hash IS NOT NULL` — the archived proxy for `html.length > 0`:
+ *   `update-page.ts` only ever writes a non-null `body_hash` on the same
+ *   `writeHtml && html.length > 0` gate the live site's `precomputedBodyHash`
+ *   computation uses, so this column faithfully reconstructs that
+ *   condition. Also excludes pre-`body_hash`-migration legacy archives'
+ *   un-backfilled rows (`migrate-page-meta-body-hash.ts` adds the column on
+ *   every writer open but does not compute values for existing rows) —
+ *   understating the replayed count is safe (never over-counts), so no
+ *   column-presence guard is needed here.
+ *
+ * Deliberately does NOT exclude rows already marked
+ * `content_items.dedupe_cap_event_id` (the post-hoc marking column) —
+ * unlike `body_hash`, that column is only backfilled during a
+ * `viewer-build`, not written during a live crawl, so filtering on it here
+ * would silently diverge from what the live crawl-time gate actually saw.
+ * `DedupeCapTracker#observe` is an O(1) no-op for a shape already in its
+ * sticky set regardless, so replaying an already-capped shape's rows costs
+ * a discarded observation, never an incorrect one.
+ *
+ * Rows are read in `ci.id` order (insertion / discovery order) via
+ * `keysetPaginateById` — Misra-Gries counting is order-dependent, so
+ * replaying in the archive's original discovery order reproduces what the
+ * tracker's state would look like had the process never restarted, subject
+ * to the same `mapCap` LRU eviction.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param onProgress - Called after each chunk, with the highest `ci.id`
+ *   scanned so far and the max `ci.id` in the table — mirrors
+ *   `getResourceUrlList`'s progress shape (`SetupProgressCallbacks.onChunkProgress`).
+ *   Omit for no reporting (tests, and callers that don't need it).
+ * @returns Every qualifying page's raw fields, in `ci.id` order.
+ */
+export async function listDedupeCapObservations(knex, onProgress) {
+    return keysetPaginateById(knex, 'content_items', (lastId) => knex('content_items as ci')
+        .join('url_refs as ur', 'ur.id', 'ci.url_id')
+        .join('page_meta as pm', 'pm.page_id', 'ci.id')
+        .leftJoin('text_refs as title_ref', 'title_ref.id', 'pm.title_text_id')
+        .leftJoin('text_refs as description_ref', 'description_ref.id', 'pm.description_text_id')
+        .leftJoin('text_refs as og_title_ref', 'og_title_ref.id', 'pm.og_title_text_id')
+        .leftJoin('url_refs as og_url_ur', 'og_url_ur.id', 'pm.og_url_id')
+        .where('ci.id', '>', lastId)
+        .andWhere('ci.scraped', 1)
+        .andWhere('ci.is_external', 0)
+        .andWhere('ci.is_target', 1)
+        .whereNull('ci.redirect_dest_id')
+        .where((qb) => {
+        qb.whereNull('ci.is_skipped').orWhere('ci.is_skipped', 0);
+    })
+        .whereNotNull('pm.body_hash')
+        .orderBy('ci.id', 'asc')
+        .limit(READ_CHUNK_SIZE)
+        .select('ci.id as id', 'ur.url as url', 'title_ref.text as title', 'description_ref.text as description', 'og_title_ref.text as ogTitle', 'og_url_ur.url as ogUrl', 'pm.body_hash as bodyHash'), (row) => ({
+        url: row.url,
+        title: row.title,
+        description: row.description,
+        ogTitle: row.ogTitle,
+        ogUrl: row.ogUrl,
+        // libsql returns BLOB columns as `Uint8Array`, not `Buffer` —
+        // `DedupeCapTracker#observe` compares hashes via `Buffer#equals`,
+        // so this must be a real `Buffer`.
+        bodyHash: Buffer.from(row.bodyHash),
+    }), onProgress);
+}

package/lib/db-ops/dedupe-cap/list-dedupe-cap-shape-keys.d.ts

@@ -0,0 +1,21 @@
+import type { Knex } from 'knex';
+/**
+ * Every distinct `dedupe_cap_events.shape_key` recorded in this archive —
+ * used by `CrawlerOrchestrator` to preload `DedupeCapTracker`'s sticky set
+ * on `--resume` / `--append` / `--retry-failed` / `--inventory`, so a trap
+ * this crawl already paid the cost of discovering once is not re-admitted
+ * in a later session. Fresh (non-resuming) crawls do not call this — there
+ * is no archive history to seed from.
+ *
+ * Unlike `listDnsBurnedHostCandidates`, no additional exclusion logic is
+ * needed: once `DedupeCapTracker` confirms a shape as a trap, it stays
+ * confirmed — there is no equivalent of "the host might have recovered
+ * since".
+ *
+ * Returns `[]` on legacy archives that pre-date the `dedupe_cap_events`
+ * table (self-healed on next writer open, so this is never a permanent
+ * state) or that have recorded no capped shapes.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns Distinct shape keys already confirmed capped.
+ */
+export declare function listDedupeCapShapeKeys(knex: Knex): Promise<string[]>;

package/lib/db-ops/dedupe-cap/list-dedupe-cap-shape-keys.js

@@ -0,0 +1,27 @@
+/**
+ * Every distinct `dedupe_cap_events.shape_key` recorded in this archive —
+ * used by `CrawlerOrchestrator` to preload `DedupeCapTracker`'s sticky set
+ * on `--resume` / `--append` / `--retry-failed` / `--inventory`, so a trap
+ * this crawl already paid the cost of discovering once is not re-admitted
+ * in a later session. Fresh (non-resuming) crawls do not call this — there
+ * is no archive history to seed from.
+ *
+ * Unlike `listDnsBurnedHostCandidates`, no additional exclusion logic is
+ * needed: once `DedupeCapTracker` confirms a shape as a trap, it stays
+ * confirmed — there is no equivalent of "the host might have recovered
+ * since".
+ *
+ * Returns `[]` on legacy archives that pre-date the `dedupe_cap_events`
+ * table (self-healed on next writer open, so this is never a permanent
+ * state) or that have recorded no capped shapes.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns Distinct shape keys already confirmed capped.
+ */
+export async function listDedupeCapShapeKeys(knex) {
+    const hasTable = await knex.schema.hasTable('dedupe_cap_events');
+    if (!hasTable) {
+        return [];
+    }
+    const rows = await knex('dedupe_cap_events').distinct('shape_key');
+    return rows.map((row) => row.shape_key);
+}

package/lib/db-ops/errors/insert-crawl-error.d.ts

@@ -0,0 +1,15 @@
+import type { Knex } from 'knex';
+/**
+ * Records a crawler-level (`error` channel) failure into `crawl_errors`.
+ *
+ * Unlike `insertPageError` this is not tied to a scraped page: `url`
+ * may be an external link that never became a page row, or `null` for a
+ * process-level error. The cause is intentionally not stored — it is derived
+ * on read so that older archives (which only have `error.log`) and freshly
+ * captured rows classify identically.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param url - The URL the error is about, or `null` for a process-level error.
+ * @param message - The error message (one line is enough for classification).
+ * @param isExternal - Whether the URL is external to the crawl scope.
+ */
+export declare function insertCrawlError(knex: Knex, url: string | null, message: string, isExternal?: boolean): Promise<void>;

package/lib/db-ops/errors/insert-crawl-error.js

@@ -0,0 +1,21 @@
+/**
+ * Records a crawler-level (`error` channel) failure into `crawl_errors`.
+ *
+ * Unlike `insertPageError` this is not tied to a scraped page: `url`
+ * may be an external link that never became a page row, or `null` for a
+ * process-level error. The cause is intentionally not stored — it is derived
+ * on read so that older archives (which only have `error.log`) and freshly
+ * captured rows classify identically.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param url - The URL the error is about, or `null` for a process-level error.
+ * @param message - The error message (one line is enough for classification).
+ * @param isExternal - Whether the URL is external to the crawl scope.
+ */
+export async function insertCrawlError(knex, url, message, isExternal = false) {
+    await knex('crawl_errors').insert({
+        url,
+        isExternal: isExternal ? 1 : 0,
+        message,
+        createdAt: Date.now(),
+    });
+}

package/lib/db-ops/errors/insert-page-error.d.ts

@@ -0,0 +1,21 @@
+import type { WriteRefCaches } from '../_shared/types.js';
+import type { Knex } from 'knex';
+/**
+ * Records a partial scrape failure against the page identified by `url`.
+ *
+ * The page row is resolved (or inserted as a stub) via
+ * {@link resolveContentItemId} so the error can be recorded even before
+ * `setPage` has run — useful when the failure fires during scraping
+ * (e.g. mid-`scrapeStart`) and the orchestrator enqueues this write
+ * before the success write for the same URL.
+ *
+ * A single page can have multiple `page_errors` rows (e.g. both
+ * `desktop-compact` and `mobile-small` viewports failing).
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param url - URL of the page being scraped.
+ * @param phase - Scrape phase name (typically `'retryExhausted'`).
+ * @param message - Human-readable failure message.
+ * @param isExternal - Whether the URL is external. Defaults to `false`.
+ */
+export declare function insertPageError(knex: Knex, caches: WriteRefCaches, url: string, phase: string, message: string, isExternal?: boolean): Promise<void>;

package/lib/db-ops/errors/insert-page-error.js

@@ -0,0 +1,30 @@
+import { resolveContentItemId } from '../_shared/resolve-content-item-id.js';
+/**
+ * Records a partial scrape failure against the page identified by `url`.
+ *
+ * The page row is resolved (or inserted as a stub) via
+ * {@link resolveContentItemId} so the error can be recorded even before
+ * `setPage` has run — useful when the failure fires during scraping
+ * (e.g. mid-`scrapeStart`) and the orchestrator enqueues this write
+ * before the success write for the same URL.
+ *
+ * A single page can have multiple `page_errors` rows (e.g. both
+ * `desktop-compact` and `mobile-small` viewports failing).
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param url - URL of the page being scraped.
+ * @param phase - Scrape phase name (typically `'retryExhausted'`).
+ * @param message - Human-readable failure message.
+ * @param isExternal - Whether the URL is external. Defaults to `false`.
+ */
+export async function insertPageError(knex, caches, url, phase, message, isExternal = false) {
+    const pageId = await resolveContentItemId(knex, caches, url, {
+        isExternal: isExternal ? 1 : 0,
+    });
+    await knex('page_errors').insert({
+        pageId,
+        phase,
+        message,
+        createdAt: Date.now(),
+    });
+}

package/lib/db-ops/errors/list-dns-burned-host-candidates.d.ts

@@ -0,0 +1,33 @@
+import type { Knex } from 'knex';
+/**
+ * Hostnames whose `crawl_errors` history is consistently DNS failures and
+ * for which no recent 2xx-3xx page or resource is recorded — i.e. hosts
+ * the previous crawl already proved unreachable. Returned in lower-cased
+ * form. Used by `CrawlerOrchestrator.#preloadDnsBurnedHostCache` so the
+ * next session short-circuits HEAD pre-flight on these hosts.
+ *
+ * Implementation: a coarse `LIKE` filter over `crawl_errors.message`
+ * narrows the row set, then `classifyErrorKind` confirms `'dns'` in JS
+ * (the regex is the single truth source — DB-side filters never narrow
+ * it). Exclusion bags are built from a single `pages` and a single
+ * `resources` scan: any host with a 2xx-3xx page, a 2xx-3xx resource, or
+ * a `pages.lastCrawledAt` newer than its latest DNS error is dropped
+ * (the host probably recovered between the failure and the last crawl).
+ *
+ * Returns `[]` on legacy archives that pre-date the `crawl_errors`
+ * table — the `hasTable` guard keeps the call non-destructive.
+ *
+ * **Outage exclusion**: a host whose latest DNS error's `createdAt` falls
+ * inside a recorded `network_outages` window is dropped from the result
+ * regardless of the other checks. A host with no prior success this
+ * archive has ever seen (the case this whole exclusion-bag scheme cannot
+ * otherwise catch) would, without this check, get preload-seeded into
+ * `dnsBurnedHostCache` and short-circuit on EVERY subsequent session
+ * forever — the exact "damage 3" this function's caller
+ * (`#preloadDnsBurnedHostCache`) exists to prevent for hosts genuinely
+ * proven dead, not to inflict on hosts merely unlucky enough to be first
+ * contacted during an operator-side network blip.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns Lower-cased hostnames safe to short-circuit.
+ */
+export declare function listDnsBurnedHostCandidates(knex: Knex): Promise<string[]>;

package/lib/db-ops/errors/list-dns-burned-host-candidates.js

@@ -0,0 +1,158 @@
+import { classifyErrorKind } from '../../error-kind/classify-error-kind.js';
+import { isWithinOutageWindow } from '../../is-within-outage-window.js';
+import { listNetworkOutages } from '../outages/list-network-outages.js';
+/**
+ * Hostnames whose `crawl_errors` history is consistently DNS failures and
+ * for which no recent 2xx-3xx page or resource is recorded — i.e. hosts
+ * the previous crawl already proved unreachable. Returned in lower-cased
+ * form. Used by `CrawlerOrchestrator.#preloadDnsBurnedHostCache` so the
+ * next session short-circuits HEAD pre-flight on these hosts.
+ *
+ * Implementation: a coarse `LIKE` filter over `crawl_errors.message`
+ * narrows the row set, then `classifyErrorKind` confirms `'dns'` in JS
+ * (the regex is the single truth source — DB-side filters never narrow
+ * it). Exclusion bags are built from a single `pages` and a single
+ * `resources` scan: any host with a 2xx-3xx page, a 2xx-3xx resource, or
+ * a `pages.lastCrawledAt` newer than its latest DNS error is dropped
+ * (the host probably recovered between the failure and the last crawl).
+ *
+ * Returns `[]` on legacy archives that pre-date the `crawl_errors`
+ * table — the `hasTable` guard keeps the call non-destructive.
+ *
+ * **Outage exclusion**: a host whose latest DNS error's `createdAt` falls
+ * inside a recorded `network_outages` window is dropped from the result
+ * regardless of the other checks. A host with no prior success this
+ * archive has ever seen (the case this whole exclusion-bag scheme cannot
+ * otherwise catch) would, without this check, get preload-seeded into
+ * `dnsBurnedHostCache` and short-circuit on EVERY subsequent session
+ * forever — the exact "damage 3" this function's caller
+ * (`#preloadDnsBurnedHostCache`) exists to prevent for hosts genuinely
+ * proven dead, not to inflict on hosts merely unlucky enough to be first
+ * contacted during an operator-side network blip.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns Lower-cased hostnames safe to short-circuit.
+ */
+export async function listDnsBurnedHostCandidates(knex) {
+    const hasCrawlErrors = await knex.schema.hasTable('crawl_errors');
+    if (!hasCrawlErrors) {
+        return [];
+    }
+    // Coarse SQL filter: cheap LIKE OR-chain over `message`. The dns regex
+    // truth source lives in `classifyErrorKind`, so we only need to feed it
+    // rows that COULD match a DNS token. Each LIKE is anchored on a known
+    // substring of the regex so future additions to the regex (without
+    // matching new SQL terms) widen the JS-side filter only — never narrow it.
+    //
+    // `%EAI_AGAIN%` is deliberately NOT in the SQL filter: it now classifies
+    // as `dns-transient` (local resolver hiccup), not `dns`, so it must not
+    // reach this candidate set. The `%getaddrinfo%` term still pulls
+    // `getaddrinfo EAI_AGAIN ...` rows but the JS-side `classifyErrorKind`
+    // check (first-match-wins) routes them to `dns-transient` and they
+    // silently drop out — keeping the cache focused on real NXDOMAIN.
+    const dnsLikeRows = (await knex('crawl_errors')
+        .select('url', 'message', 'createdAt')
+        .whereNotNull('url')
+        .where((qb) => {
+        qb.where('message', 'like', '%ENOTFOUND%')
+            .orWhere('message', 'like', '%getaddrinfo%')
+            .orWhere('message', 'like', '%ERR_NAME_NOT_RESOLVED%')
+            .orWhere('message', 'like', '%ERR_NAME_RESOLUTION_FAILED%');
+    }));
+    if (dnsLikeRows.length === 0) {
+        return [];
+    }
+    // Map<hostname, latestErrorCreatedAt> for hosts whose error message
+    // confidently classifies as DNS (LIKE matched but classifyErrorKind says
+    // e.g. `unknown` → drop).
+    const candidateLatestErrorAt = new Map();
+    for (const row of dnsLikeRows) {
+        if (classifyErrorKind(row.message) !== 'dns') {
+            continue;
+        }
+        let host;
+        try {
+            host = new URL(row.url).hostname.toLowerCase();
+        }
+        catch {
+            continue;
+        }
+        if (!host) {
+            continue;
+        }
+        const createdAt = typeof row.createdAt === 'number' ? row.createdAt : 0;
+        const previous = candidateLatestErrorAt.get(host) ?? 0;
+        if (createdAt > previous) {
+            candidateLatestErrorAt.set(host, createdAt);
+        }
+    }
+    if (candidateLatestErrorAt.size === 0) {
+        return [];
+    }
+    // Exclusion-bag #1: pages with a 2xx-3xx status anywhere on the host.
+    // Tracking the latest `last_crawled_at` per host lets us additionally
+    // drop hosts whose last successful contact post-dates the most recent
+    // DNS error (the host probably came back after a transient outage).
+    const pageOkRows = (await knex('content_items')
+        .join('url_refs', 'content_items.url_id', 'url_refs.id')
+        .select('url_refs.url as url', 'content_items.last_crawled_at as lastCrawledAt')
+        .whereBetween('content_items.status', [200, 399]));
+    const pageOkHosts = new Set();
+    const latestPageOkAt = new Map();
+    for (const row of pageOkRows) {
+        let host;
+        try {
+            host = new URL(row.url).hostname.toLowerCase();
+        }
+        catch {
+            continue;
+        }
+        pageOkHosts.add(host);
+        if (typeof row.lastCrawledAt === 'number') {
+            const previous = latestPageOkAt.get(host) ?? 0;
+            if (row.lastCrawledAt > previous) {
+                latestPageOkAt.set(host, row.lastCrawledAt);
+            }
+        }
+    }
+    // Exclusion-bag #2: non-HTML resources with a 2xx-3xx status. resources
+    // have no timestamp column so this is presence-only.
+    const resourceOkRows = (await knex('resource_items')
+        .join('url_refs', 'resource_items.url_id', 'url_refs.id')
+        .select('url_refs.url as url')
+        .whereBetween('resource_items.status', [200, 399]));
+    const resourceOkHosts = new Set();
+    for (const row of resourceOkRows) {
+        let host;
+        try {
+            host = new URL(row.url).hostname.toLowerCase();
+        }
+        catch {
+            continue;
+        }
+        resourceOkHosts.add(host);
+    }
+    const outageWindows = await listNetworkOutages(knex);
+    // A candidate host is burned only if neither pages nor resources hold a
+    // 2xx-3xx for it, its latest 2xx page (if any) is not newer than the
+    // latest DNS error (guards against re-burning a host that recovered
+    // between the last DNS failure and the most recent crawl), AND that
+    // latest DNS error did not happen during a recorded network outage.
+    const burned = [];
+    for (const [host, latestErrorAt] of candidateLatestErrorAt) {
+        if (pageOkHosts.has(host)) {
+            continue;
+        }
+        if (resourceOkHosts.has(host)) {
+            continue;
+        }
+        const latestOkAt = latestPageOkAt.get(host);
+        if (typeof latestOkAt === 'number' && latestOkAt > latestErrorAt) {
+            continue;
+        }
+        if (isWithinOutageWindow(latestErrorAt, outageWindows)) {
+            continue;
+        }
+        burned.push(host);
+    }
+    return burned;
+}

package/lib/db-ops/html/read-page-html-blob.d.ts

@@ -0,0 +1,17 @@
+import type { Knex } from 'knex';
+/**
+ * Reads the stored HTML snapshot row of a page (`page_html_ref` →
+ * `page_html_blobs`) without decoding it.
+ *
+ * Kept apart from the decode so that a caller can retry the read (a
+ * transient database failure) without retrying the decode (a corrupt or
+ * oversized blob fails the same way every time).
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId - The database ID of the page.
+ * @returns The blob's hash, stored bytes and codec, or `null` if no snapshot is stored.
+ */
+export declare function readPageHtmlBlob(knex: Knex, pageId: number): Promise<{
+    hash: Uint8Array;
+    body: Uint8Array;
+    codec: string;
+} | null>;

package/lib/db-ops/html/read-page-html-blob.js

@@ -0,0 +1,20 @@
+/**
+ * Reads the stored HTML snapshot row of a page (`page_html_ref` →
+ * `page_html_blobs`) without decoding it.
+ *
+ * Kept apart from the decode so that a caller can retry the read (a
+ * transient database failure) without retrying the decode (a corrupt or
+ * oversized blob fails the same way every time).
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId - The database ID of the page.
+ * @returns The blob's hash, stored bytes and codec, or `null` if no snapshot is stored.
+ */
+export async function readPageHtmlBlob(knex, pageId) {
+    const row = await knex
+        .from('page_html_ref')
+        .join('page_html_blobs', 'page_html_ref.hash', '=', 'page_html_blobs.hash')
+        .select('page_html_blobs.hash as hash', 'page_html_blobs.body as body', 'page_html_blobs.codec as codec')
+        .where('page_html_ref.page_id', pageId)
+        .first();
+    return row ?? null;
+}

package/lib/db-ops/lifecycle/checkpoint.d.ts

@@ -0,0 +1,8 @@
+import type { Knex } from 'knex';
+/**
+ * Forces a WAL checkpoint, writing all pending WAL data back to the main database file.
+ * Uses TRUNCATE mode to reset the WAL file to zero bytes after checkpointing.
+ * This ensures the database is fully self-contained in `db.sqlite` before archiving.
+ * @param knex - Knex query builder connected to the archive DB.
+ */
+export declare function checkpoint(knex: Knex): Promise<void>;