@nitpicker/crawler 0.11.0 → 0.13.0

package/lib/archive/db-ops/config/get-name.d.ts

@@ -0,0 +1,8 @@
+import type { Knex } from 'knex';
+/**
+ * Retrieves the crawl session name from the `info` table.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The name string.
+ * @throws {Error} If no name is found in the database.
+ */
+export declare function getName(knex: Knex): Promise<string>;

package/lib/archive/db-ops/config/get-name.js

@@ -0,0 +1,14 @@
+/**
+ * Retrieves the crawl session name from the `info` table.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The name string.
+ * @throws {Error} If no name is found in the database.
+ */
+export async function getName(knex) {
+    const selected = await knex.select('name').from('info');
+    if (!selected[0]) {
+        throw new Error('No name');
+    }
+    const [{ name }] = selected;
+    return name;
+}

package/lib/archive/db-ops/config/info-column-allowlist.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Columns of the `info` table that `setConfig` / `updateConfig` are allowed to
+ * write. Any key outside this set is silently dropped so callers can splat a
+ * wider runtime config (with extras like `cwd`) without hitting "no such
+ * column" at the SQL layer.
+ */
+export declare const INFO_COLUMN_ALLOWLIST: ReadonlySet<string>;

package/lib/archive/db-ops/config/info-column-allowlist.js

@@ -0,0 +1,26 @@
+/**
+ * Columns of the `info` table that `setConfig` / `updateConfig` are allowed to
+ * write. Any key outside this set is silently dropped so callers can splat a
+ * wider runtime config (with extras like `cwd`) without hitting "no such
+ * column" at the SQL layer.
+ */
+export const INFO_COLUMN_ALLOWLIST = new Set([
+    'version',
+    'name',
+    'baseUrl',
+    'roots',
+    'recursive',
+    'interval',
+    'image',
+    'fetchExternal',
+    'parallels',
+    'excludes',
+    'excludeKeywords',
+    'excludeUrls',
+    'maxExcludedDepth',
+    'retry',
+    'fromList',
+    'disableQueries',
+    'userAgent',
+    'ignoreRobots',
+]);

package/lib/archive/db-ops/config/info-json-columns.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Subset of {@link ./info-column-allowlist.ts} that is stored as a JSON-encoded
+ * string and therefore needs `JSON.stringify` on write.
+ */
+export declare const INFO_JSON_COLUMNS: ReadonlySet<string>;

package/lib/archive/db-ops/config/info-json-columns.js

@@ -0,0 +1,10 @@
+/**
+ * Subset of {@link ./info-column-allowlist.ts} that is stored as a JSON-encoded
+ * string and therefore needs `JSON.stringify` on write.
+ */
+export const INFO_JSON_COLUMNS = new Set([
+    'roots',
+    'excludes',
+    'excludeKeywords',
+    'excludeUrls',
+]);

package/lib/archive/db-ops/config/set-config.d.ts

@@ -0,0 +1,12 @@
+import type { Config } from '../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Stores the crawl configuration in the `info` table.
+ * Only fields in {@link ./info-column-allowlist.ts} are forwarded — any extra
+ * runtime-only field on the input is silently dropped so callers can splat
+ * a wider config object without producing SQL errors. JSON-array fields
+ * are serialized via `JSON.stringify`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param config - The {@link Config} object to store.
+ */
+export declare function setConfig(knex: Knex, config: Config): Promise<number[]>;

package/lib/archive/db-ops/config/set-config.js

@@ -0,0 +1,21 @@
+import { INFO_COLUMN_ALLOWLIST } from './info-column-allowlist.js';
+import { INFO_JSON_COLUMNS } from './info-json-columns.js';
+/**
+ * Stores the crawl configuration in the `info` table.
+ * Only fields in {@link ./info-column-allowlist.ts} are forwarded — any extra
+ * runtime-only field on the input is silently dropped so callers can splat
+ * a wider config object without producing SQL errors. JSON-array fields
+ * are serialized via `JSON.stringify`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param config - The {@link Config} object to store.
+ */
+export async function setConfig(knex, config) {
+    const payload = {};
+    for (const [key, value] of Object.entries(config)) {
+        if (!INFO_COLUMN_ALLOWLIST.has(key)) {
+            continue;
+        }
+        payload[key] = INFO_JSON_COLUMNS.has(key) ? JSON.stringify(value) : value;
+    }
+    return knex.from('info').insert(payload);
+}

package/lib/archive/db-ops/config/update-config.d.ts

@@ -0,0 +1,17 @@
+import type { Config } from '../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Update the single row in the `info` table with a partial config patch.
+ *
+ * Used by the append flow to extend `roots` (and any other tweakable
+ * field) without replacing the entire row. JSON-array fields are serialized on
+ * the fly; primitive fields are written verbatim. Unspecified fields stay as-is.
+ *
+ * Unknown keys (anything outside the allow-list of `info`-table columns) are
+ * silently dropped instead of being passed to SQL, so callers that splat a
+ * wider runtime config (e.g. `CrawlConfig` with `cwd` / `executablePath`)
+ * cannot accidentally trigger a "no such column" SQL error.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param patch - Partial {@link Config} fields to overwrite. `undefined` values are skipped.
+ */
+export declare function updateConfig(knex: Knex, patch: Partial<Config>): Promise<void>;

package/lib/archive/db-ops/config/update-config.js

@@ -0,0 +1,36 @@
+import { INFO_COLUMN_ALLOWLIST } from './info-column-allowlist.js';
+import { INFO_JSON_COLUMNS } from './info-json-columns.js';
+/**
+ * Update the single row in the `info` table with a partial config patch.
+ *
+ * Used by the append flow to extend `roots` (and any other tweakable
+ * field) without replacing the entire row. JSON-array fields are serialized on
+ * the fly; primitive fields are written verbatim. Unspecified fields stay as-is.
+ *
+ * Unknown keys (anything outside the allow-list of `info`-table columns) are
+ * silently dropped instead of being passed to SQL, so callers that splat a
+ * wider runtime config (e.g. `CrawlConfig` with `cwd` / `executablePath`)
+ * cannot accidentally trigger a "no such column" SQL error.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param patch - Partial {@link Config} fields to overwrite. `undefined` values are skipped.
+ */
+export async function updateConfig(knex, patch) {
+    const payload = {};
+    for (const [key, value] of Object.entries(patch)) {
+        if (value === undefined) {
+            continue;
+        }
+        if (!INFO_COLUMN_ALLOWLIST.has(key)) {
+            continue;
+        }
+        if (INFO_JSON_COLUMNS.has(key)) {
+            payload[key] = JSON.stringify(value);
+            continue;
+        }
+        payload[key] = value;
+    }
+    if (Object.keys(payload).length === 0) {
+        return;
+    }
+    await knex.from('info').update(payload);
+}

package/lib/archive/db-ops/errors/insert-crawl-error.d.ts

@@ -0,0 +1,15 @@
+import type { Knex } from 'knex';
+/**
+ * Records a crawler-level (`error` channel) failure into `crawl_errors`.
+ *
+ * Unlike `insertPageError` this is not tied to a scraped page: `url`
+ * may be an external link that never became a page row, or `null` for a
+ * process-level error. The cause is intentionally not stored — it is derived
+ * on read so that older archives (which only have `error.log`) and freshly
+ * captured rows classify identically.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param url - The URL the error is about, or `null` for a process-level error.
+ * @param message - The error message (one line is enough for classification).
+ * @param isExternal - Whether the URL is external to the crawl scope.
+ */
+export declare function insertCrawlError(knex: Knex, url: string | null, message: string, isExternal?: boolean): Promise<void>;

package/lib/archive/db-ops/errors/insert-crawl-error.js

@@ -0,0 +1,21 @@
+/**
+ * Records a crawler-level (`error` channel) failure into `crawl_errors`.
+ *
+ * Unlike `insertPageError` this is not tied to a scraped page: `url`
+ * may be an external link that never became a page row, or `null` for a
+ * process-level error. The cause is intentionally not stored — it is derived
+ * on read so that older archives (which only have `error.log`) and freshly
+ * captured rows classify identically.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param url - The URL the error is about, or `null` for a process-level error.
+ * @param message - The error message (one line is enough for classification).
+ * @param isExternal - Whether the URL is external to the crawl scope.
+ */
+export async function insertCrawlError(knex, url, message, isExternal = false) {
+    await knex('crawl_errors').insert({
+        url,
+        isExternal: isExternal ? 1 : 0,
+        message,
+        createdAt: Date.now(),
+    });
+}

package/lib/archive/db-ops/errors/insert-page-error.d.ts

@@ -0,0 +1,21 @@
+import type { WriteRefCaches } from '../_shared/types.js';
+import type { Knex } from 'knex';
+/**
+ * Records a partial scrape failure against the page identified by `url`.
+ *
+ * The page row is resolved (or inserted as a stub) via
+ * {@link resolveContentItemId} so the error can be recorded even before
+ * `setPage` has run — useful when the failure fires during scraping
+ * (e.g. mid-`scrapeStart`) and the orchestrator enqueues this write
+ * before the success write for the same URL.
+ *
+ * A single page can have multiple `page_errors` rows (e.g. both
+ * `desktop-compact` and `mobile-small` viewports failing).
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param url - URL of the page being scraped.
+ * @param phase - Scrape phase name (typically `'retryExhausted'`).
+ * @param message - Human-readable failure message.
+ * @param isExternal - Whether the URL is external. Defaults to `false`.
+ */
+export declare function insertPageError(knex: Knex, caches: WriteRefCaches, url: string, phase: string, message: string, isExternal?: boolean): Promise<void>;

package/lib/archive/db-ops/errors/insert-page-error.js

@@ -0,0 +1,28 @@
+import { resolveContentItemId } from '../_shared/resolve-content-item-id.js';
+/**
+ * Records a partial scrape failure against the page identified by `url`.
+ *
+ * The page row is resolved (or inserted as a stub) via
+ * {@link resolveContentItemId} so the error can be recorded even before
+ * `setPage` has run — useful when the failure fires during scraping
+ * (e.g. mid-`scrapeStart`) and the orchestrator enqueues this write
+ * before the success write for the same URL.
+ *
+ * A single page can have multiple `page_errors` rows (e.g. both
+ * `desktop-compact` and `mobile-small` viewports failing).
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param url - URL of the page being scraped.
+ * @param phase - Scrape phase name (typically `'retryExhausted'`).
+ * @param message - Human-readable failure message.
+ * @param isExternal - Whether the URL is external. Defaults to `false`.
+ */
+export async function insertPageError(knex, caches, url, phase, message, isExternal = false) {
+    const pageId = await resolveContentItemId(knex, caches, url, isExternal ? 1 : 0);
+    await knex('page_errors').insert({
+        pageId,
+        phase,
+        message,
+        createdAt: Date.now(),
+    });
+}

package/lib/archive/db-ops/errors/list-dns-burned-host-candidates.d.ts

@@ -0,0 +1,22 @@
+import type { Knex } from 'knex';
+/**
+ * Hostnames whose `crawl_errors` history is consistently DNS failures and
+ * for which no recent 2xx-3xx page or resource is recorded — i.e. hosts
+ * the previous crawl already proved unreachable. Returned in lower-cased
+ * form. Used by `CrawlerOrchestrator.#preloadDnsBurnedHostCache` so the
+ * next session short-circuits HEAD pre-flight on these hosts.
+ *
+ * Implementation: a coarse `LIKE` filter over `crawl_errors.message`
+ * narrows the row set, then `classifyErrorKind` confirms `'dns'` in JS
+ * (the regex is the single truth source — DB-side filters never narrow
+ * it). Exclusion bags are built from a single `pages` and a single
+ * `resources` scan: any host with a 2xx-3xx page, a 2xx-3xx resource, or
+ * a `pages.lastCrawledAt` newer than its latest DNS error is dropped
+ * (the host probably recovered between the failure and the last crawl).
+ *
+ * Returns `[]` on legacy archives that pre-date the `crawl_errors`
+ * table — the `hasTable` guard keeps the call non-destructive.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns Lower-cased hostnames safe to short-circuit.
+ */
+export declare function listDnsBurnedHostCandidates(knex: Knex): Promise<string[]>;

package/lib/archive/db-ops/errors/list-dns-burned-host-candidates.js

@@ -0,0 +1,141 @@
+import { classifyErrorKind } from '../../../classify-error-kind.js';
+/**
+ * Hostnames whose `crawl_errors` history is consistently DNS failures and
+ * for which no recent 2xx-3xx page or resource is recorded — i.e. hosts
+ * the previous crawl already proved unreachable. Returned in lower-cased
+ * form. Used by `CrawlerOrchestrator.#preloadDnsBurnedHostCache` so the
+ * next session short-circuits HEAD pre-flight on these hosts.
+ *
+ * Implementation: a coarse `LIKE` filter over `crawl_errors.message`
+ * narrows the row set, then `classifyErrorKind` confirms `'dns'` in JS
+ * (the regex is the single truth source — DB-side filters never narrow
+ * it). Exclusion bags are built from a single `pages` and a single
+ * `resources` scan: any host with a 2xx-3xx page, a 2xx-3xx resource, or
+ * a `pages.lastCrawledAt` newer than its latest DNS error is dropped
+ * (the host probably recovered between the failure and the last crawl).
+ *
+ * Returns `[]` on legacy archives that pre-date the `crawl_errors`
+ * table — the `hasTable` guard keeps the call non-destructive.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns Lower-cased hostnames safe to short-circuit.
+ */
+export async function listDnsBurnedHostCandidates(knex) {
+    const hasCrawlErrors = await knex.schema.hasTable('crawl_errors');
+    if (!hasCrawlErrors) {
+        return [];
+    }
+    // Coarse SQL filter: cheap LIKE OR-chain over `message`. The dns regex
+    // truth source lives in `classifyErrorKind`, so we only need to feed it
+    // rows that COULD match a DNS token. Each LIKE is anchored on a known
+    // substring of the regex so future additions to the regex (without
+    // matching new SQL terms) widen the JS-side filter only — never narrow it.
+    //
+    // `%EAI_AGAIN%` is deliberately NOT in the SQL filter: it now classifies
+    // as `dns-transient` (local resolver hiccup), not `dns`, so it must not
+    // reach this candidate set. The `%getaddrinfo%` term still pulls
+    // `getaddrinfo EAI_AGAIN ...` rows but the JS-side `classifyErrorKind`
+    // check (first-match-wins) routes them to `dns-transient` and they
+    // silently drop out — keeping the cache focused on real NXDOMAIN.
+    const dnsLikeRows = (await knex('crawl_errors')
+        .select('url', 'message', 'createdAt')
+        .whereNotNull('url')
+        .where((qb) => {
+        qb.where('message', 'like', '%ENOTFOUND%')
+            .orWhere('message', 'like', '%getaddrinfo%')
+            .orWhere('message', 'like', '%ERR_NAME_NOT_RESOLVED%')
+            .orWhere('message', 'like', '%ERR_NAME_RESOLUTION_FAILED%');
+    }));
+    if (dnsLikeRows.length === 0) {
+        return [];
+    }
+    // Map<hostname, latestErrorCreatedAt> for hosts whose error message
+    // confidently classifies as DNS (LIKE matched but classifyErrorKind says
+    // e.g. `unknown` → drop).
+    const candidateLatestErrorAt = new Map();
+    for (const row of dnsLikeRows) {
+        if (classifyErrorKind(row.message) !== 'dns') {
+            continue;
+        }
+        let host;
+        try {
+            host = new URL(row.url).hostname.toLowerCase();
+        }
+        catch {
+            continue;
+        }
+        if (!host) {
+            continue;
+        }
+        const createdAt = typeof row.createdAt === 'number' ? row.createdAt : 0;
+        const previous = candidateLatestErrorAt.get(host) ?? 0;
+        if (createdAt > previous) {
+            candidateLatestErrorAt.set(host, createdAt);
+        }
+    }
+    if (candidateLatestErrorAt.size === 0) {
+        return [];
+    }
+    // Exclusion-bag #1: pages with a 2xx-3xx status anywhere on the host.
+    // Tracking the latest `last_crawled_at` per host lets us additionally
+    // drop hosts whose last successful contact post-dates the most recent
+    // DNS error (the host probably came back after a transient outage).
+    const pageOkRows = (await knex('content_items')
+        .join('url_refs', 'content_items.url_id', 'url_refs.id')
+        .select('url_refs.url as url', 'content_items.last_crawled_at as lastCrawledAt')
+        .whereBetween('content_items.status', [200, 399]));
+    const pageOkHosts = new Set();
+    const latestPageOkAt = new Map();
+    for (const row of pageOkRows) {
+        let host;
+        try {
+            host = new URL(row.url).hostname.toLowerCase();
+        }
+        catch {
+            continue;
+        }
+        pageOkHosts.add(host);
+        if (typeof row.lastCrawledAt === 'number') {
+            const previous = latestPageOkAt.get(host) ?? 0;
+            if (row.lastCrawledAt > previous) {
+                latestPageOkAt.set(host, row.lastCrawledAt);
+            }
+        }
+    }
+    // Exclusion-bag #2: non-HTML resources with a 2xx-3xx status. resources
+    // have no timestamp column so this is presence-only.
+    const resourceOkRows = (await knex('resource_items')
+        .join('url_refs', 'resource_items.url_id', 'url_refs.id')
+        .select('url_refs.url as url')
+        .whereBetween('resource_items.status', [200, 399]));
+    const resourceOkHosts = new Set();
+    for (const row of resourceOkRows) {
+        let host;
+        try {
+            host = new URL(row.url).hostname.toLowerCase();
+        }
+        catch {
+            continue;
+        }
+        resourceOkHosts.add(host);
+    }
+    // A candidate host is burned only if neither pages nor resources hold a
+    // 2xx-3xx for it, AND its latest 2xx page (if any) is not newer than
+    // the latest DNS error. The third check guards against re-burning a
+    // host that recovered between the last DNS failure and the most recent
+    // crawl.
+    const burned = [];
+    for (const [host, latestErrorAt] of candidateLatestErrorAt) {
+        if (pageOkHosts.has(host)) {
+            continue;
+        }
+        if (resourceOkHosts.has(host)) {
+            continue;
+        }
+        const latestOkAt = latestPageOkAt.get(host);
+        if (typeof latestOkAt === 'number' && latestOkAt > latestErrorAt) {
+            continue;
+        }
+        burned.push(host);
+    }
+    return burned;
+}

package/lib/archive/db-ops/html/get-html-of-page-by-id.d.ts

@@ -0,0 +1,18 @@
+import type { Knex } from 'knex';
+/**
+ * Reads the HTML snapshot stored as a zstd-compressed BLOB for the given page.
+ *
+ * Joins `page_html_ref` → `page_html_blobs` and decompresses inline. Returns
+ * `null` when the page has no stored body (a non-HTML resource, a redirect
+ * source, a degraded render). Read works identically on read-only / stub
+ * connections — the special-cased "do we have a loose dir vs zip?" branching
+ * the previous file-backed layout required is gone.
+ *
+ * Tables `page_html_ref` and `page_html_blobs` are created by `initSchema`.
+ * Older `.nitpicker` archives that predate this migration must be passed
+ * through `scripts/migrate-to-0.10.mjs` before they can be read.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId - The database ID of the page.
+ * @returns The decompressed HTML string, or `null` if no snapshot is stored.
+ */
+export declare function getHtmlOfPageById(knex: Knex, pageId: number): Promise<string | null>;

package/lib/archive/db-ops/html/get-html-of-page-by-id.js

@@ -0,0 +1,29 @@
+import { decodeStoredBlob } from '../../decode-html-blob.js';
+/**
+ * Reads the HTML snapshot stored as a zstd-compressed BLOB for the given page.
+ *
+ * Joins `page_html_ref` → `page_html_blobs` and decompresses inline. Returns
+ * `null` when the page has no stored body (a non-HTML resource, a redirect
+ * source, a degraded render). Read works identically on read-only / stub
+ * connections — the special-cased "do we have a loose dir vs zip?" branching
+ * the previous file-backed layout required is gone.
+ *
+ * Tables `page_html_ref` and `page_html_blobs` are created by `initSchema`.
+ * Older `.nitpicker` archives that predate this migration must be passed
+ * through `scripts/migrate-to-0.10.mjs` before they can be read.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId - The database ID of the page.
+ * @returns The decompressed HTML string, or `null` if no snapshot is stored.
+ */
+export async function getHtmlOfPageById(knex, pageId) {
+    const row = await knex
+        .from('page_html_ref')
+        .join('page_html_blobs', 'page_html_ref.hash', '=', 'page_html_blobs.hash')
+        .select('page_html_blobs.body as body', 'page_html_blobs.codec as codec')
+        .where('page_html_ref.page_id', pageId)
+        .first();
+    if (!row) {
+        return null;
+    }
+    return decodeStoredBlob(row.body, row.codec);
+}

package/lib/archive/db-ops/inventory/record-inventory-run.d.ts

@@ -0,0 +1,21 @@
+import type { InventoryRunMeta } from '../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Appends one row to the `inventory_runs` audit log.
+ *
+ * Called by `CrawlerOrchestrator.inventory` on every successful
+ * `--inventory <list>` invocation so the archive carries a durable
+ * record of which deploy list was applied when and at what scale —
+ * the operational question "did we apply last month's list" the
+ * archive itself can answer without consulting external bookkeeping.
+ *
+ * Append-only. There is intentionally no UPDATE path and no UNIQUE
+ * constraint on `source_file_sha256`; two applies of the same list
+ * each get their own row. Duplicate detection is left to readers —
+ * the hash is recorded as the content-identity key they would use.
+ * Field-level NULL semantics live on {@link InventoryRunMeta}.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param meta - The run metadata to record. Only `ran_at` is required.
+ * @returns The autoincremented `id` of the newly-inserted row.
+ */
+export declare function recordInventoryRun(knex: Knex, meta: InventoryRunMeta): Promise<number>;

package/lib/archive/db-ops/inventory/record-inventory-run.js

@@ -0,0 +1,38 @@
+/**
+ * Appends one row to the `inventory_runs` audit log.
+ *
+ * Called by `CrawlerOrchestrator.inventory` on every successful
+ * `--inventory <list>` invocation so the archive carries a durable
+ * record of which deploy list was applied when and at what scale —
+ * the operational question "did we apply last month's list" the
+ * archive itself can answer without consulting external bookkeeping.
+ *
+ * Append-only. There is intentionally no UPDATE path and no UNIQUE
+ * constraint on `source_file_sha256`; two applies of the same list
+ * each get their own row. Duplicate detection is left to readers —
+ * the hash is recorded as the content-identity key they would use.
+ * Field-level NULL semantics live on {@link InventoryRunMeta}.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param meta - The run metadata to record. Only `ran_at` is required.
+ * @returns The autoincremented `id` of the newly-inserted row.
+ */
+export async function recordInventoryRun(knex, meta) {
+    const inserted = await knex
+        .from('inventory_runs')
+        .insert({
+        ran_at: meta.ran_at,
+        list_label: meta.list_label ?? null,
+        source_file_sha256: meta.source_file_sha256 ?? null,
+        total_lines: meta.total_lines ?? null,
+        new_pages: meta.new_pages ?? null,
+        new_resources: meta.new_resources ?? null,
+        scope_skipped: meta.scope_skipped ?? null,
+        notes: meta.notes ?? null,
+    })
+        .returning('id');
+    const id = inserted[0]?.id;
+    if (typeof id !== 'number') {
+        throw new TypeError('recordInventoryRun: INSERT returned no row id');
+    }
+    return id;
+}

package/lib/archive/db-ops/lifecycle/checkpoint.d.ts

@@ -0,0 +1,8 @@
+import type { Knex } from 'knex';
+/**
+ * Forces a WAL checkpoint, writing all pending WAL data back to the main database file.
+ * Uses TRUNCATE mode to reset the WAL file to zero bytes after checkpointing.
+ * This ensures the database is fully self-contained in `db.sqlite` before archiving.
+ * @param knex - Knex query builder connected to the archive DB.
+ */
+export declare function checkpoint(knex: Knex): Promise<void>;

package/lib/archive/db-ops/lifecycle/checkpoint.js

@@ -0,0 +1,9 @@
+/**
+ * Forces a WAL checkpoint, writing all pending WAL data back to the main database file.
+ * Uses TRUNCATE mode to reset the WAL file to zero bytes after checkpointing.
+ * This ensures the database is fully self-contained in `db.sqlite` before archiving.
+ * @param knex - Knex query builder connected to the archive DB.
+ */
+export async function checkpoint(knex) {
+    await knex.raw('PRAGMA wal_checkpoint(TRUNCATE)');
+}

package/lib/archive/db-ops/lifecycle/destroy.d.ts

@@ -0,0 +1,6 @@
+import type { Knex } from 'knex';
+/**
+ * Destroys the database connection, releasing all pooled resources.
+ * @param knex - Knex query builder connected to the archive DB.
+ */
+export declare function destroy(knex: Knex): Promise<void>;

package/lib/archive/db-ops/lifecycle/destroy.js

@@ -0,0 +1,7 @@
+/**
+ * Destroys the database connection, releasing all pooled resources.
+ * @param knex - Knex query builder connected to the archive DB.
+ */
+export async function destroy(knex) {
+    await knex.destroy();
+}

package/lib/archive/db-ops/lifecycle/init.d.ts

@@ -0,0 +1,22 @@
+import type { Knex } from 'knex';
+/**
+ * Initializes the database schema if tables do not exist, then runs the
+ * one remaining lightweight migration (`info.roots`).
+ *
+ * There is deliberately no per-table lazy-migration chain here:
+ * `assertCompatibleVersion` (called below, before any schema work)
+ * rejects every archive older than the current format, so a connection
+ * that reaches `initSchema` is either brand new (initSchema provisions
+ * the full schema) or was produced by `scripts/migrate-to-0.13.mjs`
+ * (which guarantees the full table set before it repacks). A
+ * `hasTable`-guarded catch-up migration could therefore never fire —
+ * schema catch-up for old archives is the migration script's job, not
+ * the open path's.
+ *
+ * In read-only mode schema init + migration are SKIPPED so the same DB
+ * can be opened safely by a viewer attached to a live (or interrupted)
+ * crawl without rewriting the user's tmpDir.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param readOnly - When true, skip schema init + migrations.
+ */
+export declare function init(knex: Knex, readOnly: boolean): Promise<void>;

package/lib/archive/db-ops/lifecycle/init.js

@@ -0,0 +1,42 @@
+import { applyConnectionPragmas, initSchema } from '../../init-schema.js';
+import { assertCompatibleVersion } from '../../meta/assert-compatible-version.js';
+import { migrateInfoRoots } from '../../migrate-info-roots.js';
+/**
+ * Initializes the database schema if tables do not exist, then runs the
+ * one remaining lightweight migration (`info.roots`).
+ *
+ * There is deliberately no per-table lazy-migration chain here:
+ * `assertCompatibleVersion` (called below, before any schema work)
+ * rejects every archive older than the current format, so a connection
+ * that reaches `initSchema` is either brand new (initSchema provisions
+ * the full schema) or was produced by `scripts/migrate-to-0.13.mjs`
+ * (which guarantees the full table set before it repacks). A
+ * `hasTable`-guarded catch-up migration could therefore never fire —
+ * schema catch-up for old archives is the migration script's job, not
+ * the open path's.
+ *
+ * In read-only mode schema init + migration are SKIPPED so the same DB
+ * can be opened safely by a viewer attached to a live (or interrupted)
+ * crawl without rewriting the user's tmpDir.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param readOnly - When true, skip schema init + migrations.
+ */
+export async function init(knex, readOnly) {
+    // Connection-level PRAGMAs (foreign_keys, mmap_size, …) must be
+    // reapplied on every connect — they are not persisted across opens.
+    // They are safe in read-only mode because they don't write to the
+    // user's tmpDir, just configure the libsql connection.
+    await applyConnectionPragmas(knex);
+    // Reject incompatible archives before any further work. Runs for both
+    // writer and read-only (stub viewer) connections so old
+    // `._nitpicker-*` stubs surface a clear error instead of
+    // dereferencing missing columns at query time. New archives (no
+    // `info` table yet) pass through; the schema is filled in by
+    // `initSchema` below.
+    await assertCompatibleVersion(knex);
+    if (readOnly) {
+        return;
+    }
+    await initSchema(knex);
+    await migrateInfoRoots(knex);
+}