@nitpicker/crawler 0.11.0 → 0.12.0

package/lib/archive/derive-lineage-from-parent.d.ts

@@ -0,0 +1,37 @@
+import type { PageSource } from './types.js';
+/**
+ * Decide which {@link PageSource} label a child row reached through the
+ * crawl graph (anchor placeholder, sub-resource, redirect chain
+ * intermediate) should inherit from its parent.
+ *
+ * Two simple rules, expressed once so anchor / redirect / sub-resource
+ * call sites stay in lockstep:
+ *
+ * 1. If the parent is in the inventory chain
+ *    ({@link isInventorySource}) → propagate
+ *    `'inventory-discovered'`. The child is itself a transitively
+ *    reached node in the inventory chain; it is NOT a new seed (the
+ *    seed label is reserved for URLs the operator listed in
+ *    `--inventory ./list.txt`).
+ *
+ * 2. Otherwise → return `fallback`. The two production fallbacks differ
+ *    by call site:
+ *
+ *    - Anchor lineage passes `'crawled'` explicitly so the crawled-wins
+ *      downgrade inside `#getIdByUrl` fires when the anchor reaches an
+ *      existing `'inventory-*'` row.
+ *    - Sub-resource emit passes `undefined` so the DB DEFAULT
+ *      `'crawled'` lands on the freshly INSERTed `resources` row (the
+ *      `setResources` path is INSERT-only with `onConflict.ignore()`, so
+ *      no downgrade is needed).
+ *    - Redirect chain intermediate uses `'crawled'` (same reason as
+ *      anchor): an existing inventory-* intermediate reached by a
+ *      crawled redirect chain must be downgraded.
+ *
+ * Pure function — keeps the lineage decision testable in isolation from
+ * the database transaction / event-emitter wiring that consumes it.
+ * @param parentSource - The parent page's stored `source` column (or `undefined` when no parent row exists).
+ * @param fallback - The label to return when the parent is NOT in the inventory chain. Pass `'crawled'` to enable the crawled-wins downgrade, or `undefined` to let the DB DEFAULT apply.
+ * @returns The lineage label to attach to the child row.
+ */
+export declare function deriveLineageFromParent(parentSource: PageSource | undefined, fallback: PageSource | undefined): PageSource | undefined;

package/lib/archive/derive-lineage-from-parent.js

@@ -0,0 +1,42 @@
+import { isInventorySource } from './is-inventory-source.js';
+/**
+ * Decide which {@link PageSource} label a child row reached through the
+ * crawl graph (anchor placeholder, sub-resource, redirect chain
+ * intermediate) should inherit from its parent.
+ *
+ * Two simple rules, expressed once so anchor / redirect / sub-resource
+ * call sites stay in lockstep:
+ *
+ * 1. If the parent is in the inventory chain
+ *    ({@link isInventorySource}) → propagate
+ *    `'inventory-discovered'`. The child is itself a transitively
+ *    reached node in the inventory chain; it is NOT a new seed (the
+ *    seed label is reserved for URLs the operator listed in
+ *    `--inventory ./list.txt`).
+ *
+ * 2. Otherwise → return `fallback`. The two production fallbacks differ
+ *    by call site:
+ *
+ *    - Anchor lineage passes `'crawled'` explicitly so the crawled-wins
+ *      downgrade inside `#getIdByUrl` fires when the anchor reaches an
+ *      existing `'inventory-*'` row.
+ *    - Sub-resource emit passes `undefined` so the DB DEFAULT
+ *      `'crawled'` lands on the freshly INSERTed `resources` row (the
+ *      `setResources` path is INSERT-only with `onConflict.ignore()`, so
+ *      no downgrade is needed).
+ *    - Redirect chain intermediate uses `'crawled'` (same reason as
+ *      anchor): an existing inventory-* intermediate reached by a
+ *      crawled redirect chain must be downgraded.
+ *
+ * Pure function — keeps the lineage decision testable in isolation from
+ * the database transaction / event-emitter wiring that consumes it.
+ * @param parentSource - The parent page's stored `source` column (or `undefined` when no parent row exists).
+ * @param fallback - The label to return when the parent is NOT in the inventory chain. Pass `'crawled'` to enable the crawled-wins downgrade, or `undefined` to let the DB DEFAULT apply.
+ * @returns The lineage label to attach to the child row.
+ */
+export function deriveLineageFromParent(parentSource, fallback) {
+    if (isInventorySource(parentSource)) {
+        return 'inventory-discovered';
+    }
+    return fallback;
+}

package/lib/archive/get-failed-page-messages.d.ts

@@ -0,0 +1,43 @@
+import type { Knex } from 'knex';
+/**
+ * Bulk-resolve a raw error message for each given page id, using only sources
+ * reachable from a {@link Knex} handle. Read order: `page_errors` (keyed by
+ * `pageId`, the most direct signal a scrape attempt recorded), then
+ * `crawl_errors` (keyed by `url`, the crawler-channel record for failures
+ * that happened before a page row was scraped).
+ *
+ * **Known limitation — pre-`crawl_errors` archives**: This helper does NOT
+ * read `error.log`. `migrateCrawlErrors` creates the `crawl_errors` table on
+ * writer connect but does NOT back-fill historical lines from `error.log`,
+ * so an archive that predates the `crawl_errors` schema and whose failures
+ * live only in `error.log` will resolve every id to "no message" here. The
+ * downstream `Database.resetFailedPages` treats absence as `unknown` (still
+ * retryable), so legacy archives lose the permanent-kind exclusion until a
+ * fresh crawl run populates `crawl_errors` / `page_errors`. The trade-off
+ * (no error.log parsing in the writer path) keeps the writer dependency
+ * surface narrow and avoids re-implementing the parser already living in
+ * `@nitpicker/query`'s `resolveFailedPageMessages` — which the crawler
+ * package cannot import (reverse-direction dependency). When this matters
+ * in practice, run the archive through one fresh `crawl --retry-failed`
+ * pass first to populate the structured tables.
+ *
+ * Pages with no message in any consulted source are simply absent from the
+ * returned map; callers treat the absence as "unclassifiable, keep retrying"
+ * (i.e. `unknown`).
+ * @param instance - The {@link Knex} handle.
+ * @param ids - Candidate `pages.id` values.
+ * @param urls - The corresponding `pages.url` values, in the same order as
+ *   `ids`. Length and indexing MUST match `ids` so the page → url join can be
+ *   reconstructed without a second `pages` round-trip.
+ * @returns `Map<pageId, message>` populated only for ids whose message was
+ *   found in one of the consulted tables.
+ * @example
+ * ```ts
+ * const messages = await getFailedPageMessages(
+ *   instance,
+ *   candidates.map(c => c.id),
+ *   candidates.map(c => c.url),
+ * );
+ * ```
+ */
+export declare function getFailedPageMessages(instance: Knex, ids: readonly number[], urls: readonly string[]): Promise<Map<number, string>>;

package/lib/archive/get-failed-page-messages.js

@@ -0,0 +1,131 @@
+/** chunk size for `WHERE … IN (?)` SQLite parameter packing. */
+const CHUNK_SIZE = 500;
+/**
+ * Bulk-resolve a raw error message for each given page id, using only sources
+ * reachable from a {@link Knex} handle. Read order: `page_errors` (keyed by
+ * `pageId`, the most direct signal a scrape attempt recorded), then
+ * `crawl_errors` (keyed by `url`, the crawler-channel record for failures
+ * that happened before a page row was scraped).
+ *
+ * **Known limitation — pre-`crawl_errors` archives**: This helper does NOT
+ * read `error.log`. `migrateCrawlErrors` creates the `crawl_errors` table on
+ * writer connect but does NOT back-fill historical lines from `error.log`,
+ * so an archive that predates the `crawl_errors` schema and whose failures
+ * live only in `error.log` will resolve every id to "no message" here. The
+ * downstream `Database.resetFailedPages` treats absence as `unknown` (still
+ * retryable), so legacy archives lose the permanent-kind exclusion until a
+ * fresh crawl run populates `crawl_errors` / `page_errors`. The trade-off
+ * (no error.log parsing in the writer path) keeps the writer dependency
+ * surface narrow and avoids re-implementing the parser already living in
+ * `@nitpicker/query`'s `resolveFailedPageMessages` — which the crawler
+ * package cannot import (reverse-direction dependency). When this matters
+ * in practice, run the archive through one fresh `crawl --retry-failed`
+ * pass first to populate the structured tables.
+ *
+ * Pages with no message in any consulted source are simply absent from the
+ * returned map; callers treat the absence as "unclassifiable, keep retrying"
+ * (i.e. `unknown`).
+ * @param instance - The {@link Knex} handle.
+ * @param ids - Candidate `pages.id` values.
+ * @param urls - The corresponding `pages.url` values, in the same order as
+ *   `ids`. Length and indexing MUST match `ids` so the page → url join can be
+ *   reconstructed without a second `pages` round-trip.
+ * @returns `Map<pageId, message>` populated only for ids whose message was
+ *   found in one of the consulted tables.
+ * @example
+ * ```ts
+ * const messages = await getFailedPageMessages(
+ *   instance,
+ *   candidates.map(c => c.id),
+ *   candidates.map(c => c.url),
+ * );
+ * ```
+ */
+export async function getFailedPageMessages(instance, ids, urls) {
+    if (ids.length === 0) {
+        return new Map();
+    }
+    if (ids.length !== urls.length) {
+        throw new Error(`getFailedPageMessages: ids.length (${ids.length}) !== urls.length (${urls.length}) — must be 1:1`);
+    }
+    const messageByPageId = new Map();
+    if (await instance.schema.hasTable('page_errors')) {
+        for (let i = 0; i < ids.length; i += CHUNK_SIZE) {
+            const chunk = ids.slice(i, i + CHUNK_SIZE);
+            // `orderBy('id', 'asc')` makes the per-pageId "first row seen"
+            // behavior deterministic regardless of SQLite's natural ROWID
+            // scan order, which is otherwise implementation-defined under
+            // concurrent / migrated archives. Without the explicit order, a
+            // reset that classified a page as `unknown` once could classify
+            // it as `parse-error` on the next run when the rows happen to be
+            // returned in a different order.
+            const rows = (await instance('page_errors')
+                .select('pageId', 'message')
+                .whereIn('pageId', chunk)
+                .orderBy('id', 'asc'));
+            for (const row of rows) {
+                // Earliest-id wins. Schema permits multiple rows per pageId
+                // (the same scrape can record several phase errors); the
+                // first row inserted is usually the trigger cause, later
+                // rows are follow-on noise from the same failure cascade.
+                //
+                // An empty `message` is treated as "no signal" and ignored
+                // so the crawl_errors lookup can fill it in. Without this,
+                // a page_errors row with `message=''` (recorded by a
+                // scraper phase that fired its trigger but had no error
+                // text) would short-circuit and we'd lose access to the
+                // crawl_errors row that classifies the failure as
+                // `dns` / `tls` / `client-blocked` etc. — defeating
+                // `--retry-failed`'s permanent-kind exclusion.
+                if (row.message !== '' && !messageByPageId.has(row.pageId)) {
+                    messageByPageId.set(row.pageId, row.message);
+                }
+            }
+        }
+    }
+    const idsMissing = ids.filter((id) => !messageByPageId.has(id));
+    if (idsMissing.length === 0 || !(await instance.schema.hasTable('crawl_errors'))) {
+        // Early-exit short-circuits BOTH the idToUrl Map construction and
+        // the crawl_errors round-trip. On a 1M-page archive where every
+        // failed page already has a `page_errors` row, this avoids walking
+        // the candidate list a second time.
+        return messageByPageId;
+    }
+    const idToUrl = new Map();
+    for (const [i, id] of ids.entries()) {
+        const url = urls[i];
+        if (url !== undefined) {
+            idToUrl.set(id, url);
+        }
+    }
+    const missingUrls = [];
+    for (const id of idsMissing) {
+        const url = idToUrl.get(id);
+        if (url !== undefined) {
+            missingUrls.push(url);
+        }
+    }
+    const urlToMessage = new Map();
+    for (let i = 0; i < missingUrls.length; i += CHUNK_SIZE) {
+        const chunk = missingUrls.slice(i, i + CHUNK_SIZE);
+        const rows = (await instance('crawl_errors')
+            .select('url', 'message')
+            .whereIn('url', chunk));
+        for (const row of rows) {
+            if (row.url !== null && !urlToMessage.has(row.url)) {
+                urlToMessage.set(row.url, row.message);
+            }
+        }
+    }
+    for (const id of idsMissing) {
+        const url = idToUrl.get(id);
+        if (url === undefined) {
+            continue;
+        }
+        const message = urlToMessage.get(url);
+        if (message !== undefined) {
+            messageByPageId.set(id, message);
+        }
+    }
+    return messageByPageId;
+}

package/lib/archive/init-schema.js

@@ -15,9 +15,20 @@ export async function applyConnectionPragmas(instance) {
     await instance.raw('PRAGMA foreign_keys = ON');
     await instance.raw('PRAGMA wal_autocheckpoint = 1000');
     // Negative value = KiB of memory (64 MiB). Helps large BLOB scans.
+    //
+    // Empirically validated against larger values on a 10 GB archive:
+    // bumping to 512 MiB regressed `getSummary` (1.9s → 5.7s), `pages`
+    // (2.3s → 21s), and `images` (3.7s → 12s) — libsql's page eviction
+    // policy interacts poorly with a cache sized comparable to the
+    // host's page-cache window when the DB itself far exceeds RAM.
+    // 64 MiB stays the sweet spot.
     await instance.raw('PRAGMA cache_size = -65536');
     // 256 MiB mmap window. SQLite falls back to read() past this so the
-    // limit is a soft ceiling, not a hard one.
+    // limit is a soft ceiling, not a hard one. A 4 GiB window was
+    // catastrophic on a 10 GB archive on macOS (summary 1.9s → 43s,
+    // pages 2.3s → 21s) — the kernel's read-ahead policy and libsql's
+    // mmap path interact badly when the window can cover most of the
+    // DB. Keep this conservative.
     await instance.raw('PRAGMA mmap_size = 268435456');
 }
 /**
@@ -318,6 +329,23 @@ export async function initSchema(instance) {
         t.text('parseError');
         t.index('pageId');
         t.index('type');
+    })
+        .createTable('inventory_runs', (t) => {
+        // One row per successful `--inventory <list>` invocation. The
+        // archive's audit log of "when did we apply which deploy list
+        // at what scale". `.bak` is removed on success so this table
+        // is the only durable provenance record. Schema rationale +
+        // non-goals live in {@link migrateInventoryRuns}.
+        t.increments('id');
+        t.string('ran_at').notNullable();
+        t.string('list_label').nullable();
+        t.string('source_file_sha256', 64).nullable();
+        t.integer('total_lines').nullable();
+        t.integer('new_pages').nullable();
+        t.integer('new_resources').nullable();
+        t.integer('scope_skipped').nullable();
+        t.text('notes').nullable();
+        t.index('ran_at');
     });
     // ON DELETE CASCADE and compound indexes for the new tables. Knex's
     // schema builder can't express CASCADE / compound indexes inline in a
@@ -348,4 +376,128 @@ export async function initSchema(instance) {
 		) WITHOUT ROWID
 	`);
     await instance.raw('CREATE INDEX idx_page_html_ref_hash ON page_html_ref(hash)');
+    // Composite covering index for the default Pages-view filter + url-ordered
+    // scan. Without it, `listPages` on a 400k-row archive runs ~15s per page
+    // click (SCAN pages USING pages_scraped_index + TEMP B-TREE FOR ORDER BY);
+    // with it, the same query runs ~45ms (368x speedup, confirmed via
+    // `scripts/bench-partial-listfilter.mjs` against a real customer archive).
+    // The same index also serves `listIsolatedPages`, `listIsolatedClusters`,
+    // and `getSummary`'s HTML-page counts.
+    //
+    // **Column order: `(isExternal, scraped, redirectDestId, url, contentType)`.**
+    // The leading `isExternal` is critical: the Pages view's default
+    // "external excluded" filter adds `WHERE isExternal = 0` to both the
+    // SELECT and the paginate-query COUNT. A previous version of this index
+    // (`(scraped, redirectDestId, url, contentType)`) shipped without
+    // `isExternal`, and the SELECT picked it up (`ORDER BY url` forced the
+    // match) while the COUNT — having no `ORDER BY` — fell back to the
+    // single-column `pages_isexternal_index` + scan + per-row WHERE filter,
+    // costing ~8.7s for the COUNT alone on a 165k-internal-page archive.
+    // Putting `isExternal` first makes both shapes pick this index as a
+    // covering scan (~33ms COUNT, ~1ms SELECT warm).
+    //
+    // **DO NOT RUN `ANALYZE` ON .nitpicker ARCHIVES.** With ANALYZE statistics
+    // available, the planner switches the JOIN paths in `listLinks`,
+    // `getLinkGraph`, and `listPageLinks` to use this index for source/dest
+    // seeks (SCAN dest → SEARCH anchors → SEARCH source) instead of the
+    // existing `SCAN anchors → rowid seek` plan. That regression takes those
+    // queries from ~15s to ~500s (33x worse). The unanalyzed-table heuristic
+    // happens to pick the right plan for the joins while still picking the new
+    // index for `listPages` because the column order exactly matches the
+    // WHERE+ORDER predicates. If a future change adds `ANALYZE` anywhere in
+    // the crawler / viewer / MCP / migration paths, this index must be
+    // re-evaluated first.
+    await instance.raw('CREATE INDEX idx_pages_listfilter ON pages(isExternal, scraped, redirectDestId, url, contentType)');
+    // Covering index for `listUnusedResources`. Without it the query SCAN s
+    // `resources_url_unique` (every resource, including externals) then
+    // filters `isExternal = 0` row-by-row — ~66s on the bench archive. With
+    // the `(isExternal, url)` leading prefix, the planner serves the WHERE
+    // + ORDER BY url from one covering scan — ~7.5s (8.8x). Same
+    // no-ANALYZE invariant applies (see `idx_pages_listfilter` above);
+    // validated against the 4 regression sentinels in
+    // `scripts/bench-unused-images.mjs`.
+    await instance.raw('CREATE INDEX idx_resources_internal_url ON resources(isExternal, url)');
+    // Covering index for `listImages`. The default query joins `images` to
+    // `pages` and orders by `pages.url`. Without this index the planner
+    // scans `images` first, seeks `pages` by rowid, and pays a TEMP B-TREE
+    // FOR ORDER BY (~32s on the bench archive). With the index the plan
+    // flips to SCAN pages (via `pages_url_unique`, url-ordered already)
+    // → SEARCH images via the covering pageId index — no temp sort, ~16s
+    // (2.0x). The included columns (src, alt, dimensions, isLazy) make
+    // `idx_images_covering` covering for every `select` `listImages` does,
+    // so the SEARCH does not need to materialise the underlying row.
+    // Validated by `scripts/bench-unused-images.mjs`.
+    await instance.raw('CREATE INDEX idx_images_covering ON images(pageId, src, alt, width, height, naturalWidth, naturalHeight, isLazy)');
+    // Targets `getSummary` Q2 (metadata fulfilment) + Q3 (content-type
+    // histogram). With this index Q2 and Q3 both become covering
+    // (`SEARCH ... USING COVERING INDEX`) — the SELECT columns are
+    // contained inside the index entry, so no rowid lookup is needed.
+    // Q1 (status histogram) also picks this index for its seek but
+    // keeps `USE TEMP B-TREE FOR GROUP BY` because the index column
+    // order leads with `contentType, isExternal` while Q1's GROUP BY
+    // is `(isExternal, status)` — the residual ordering inside the
+    // `scraped=1 AND redirectDestId IS NULL` slice does not match.
+    // Empirically this still gives the largest net win because Q2 +
+    // Q3 dominate `getSummary` on archives whose `pages` table dwarfs
+    // the SQLite page cache (10 GB bench: 1157 ms → 717 ms, 38 %).
+    //
+    // **An additional candidate index `(scraped, redirectDestId,
+    // isExternal, status)` was empirically rejected**: in isolation it
+    // matches Q1's GROUP BY column order and would eliminate the temp
+    // B-tree there, but in combination with this one or with the
+    // `pages_scraped_index` fallback the planner shifted to plans that
+    // regressed `getSummary` to 4.6-10 s (PR #96 教訓 — bulk index
+    // addition without ANALYZE confuses the heuristic). The
+    // `idx_pages_summary_contenttype` form below is the only summary
+    // index that survived the matrix test in
+    // `scripts/bench-summary-configs.mjs`.
+    //
+    // Column order rationale:
+    //
+    // 1. `scraped` — leading seek key. All summary queries constrain
+    //    it to `=1`.
+    // 2. `redirectDestId` — post-seek filter, IS NULL folded into the
+    //    seek key by SQLite's index walk without needing ANALYZE
+    //    (per operator forum; the IS NULL leading column rule only
+    //    bites when the column is the LEADING one and there is no
+    //    other equality constraint).
+    // 3. `contentType` — the column Q3 groups by.
+    // 4. `isExternal` — Q3's second GROUP BY column AND Q2's WHERE
+    //    constraint (`isExternal=0`).
+    // 5. `isSkipped` — Q1/Q3's residual `(isSkipped=0 OR IS NULL)`
+    //    filter (`excludeSkippedPages`). Having it in the index lets
+    //    the residual filter use the index entry instead of a per-row
+    //    rowid lookup.
+    //
+    // `id` is implicitly included (every SQLite index entry carries
+    // the rowid), so the `count(id)` aggregates cover off-index.
+    //
+    // **No-ANALYZE invariant** identical to `idx_pages_listfilter`
+    // (PR #96): the column order matches the WHERE+GROUP BY predicates
+    // exactly, so SQLite's heuristic-only planner picks it without
+    // needing `sqlite_stat1`. Adding `ANALYZE` would risk planner
+    // shifts in this and other queries.
+    //
+    // **Regression check**: `listPages` / `listPages COUNT` /
+    // `listLinks broken` / `listPageLinks` plans were re-verified
+    // against this index — `idx_pages_listfilter` continues to win
+    // for all of them. See `scripts/bench-summary-configs.mjs`.
+    await instance.raw('CREATE INDEX idx_pages_summary_contenttype ON pages(scraped, redirectDestId, contentType, isExternal, isSkipped)');
+    // Targets `getSummary` Q4 (`failedPageIdRows`) — selects pages with
+    // `scraped=1 AND status=-1 AND redirectDestId IS NULL`. `status=-1`
+    // is highly selective (a few hundred rows on archives with
+    // ~400 k `scraped=1` pages), but without this index the planner
+    // seeks all `scraped=1` rows via `pages_scraped_index` and then
+    // row-by-row filters status, costing ~5 s. The 3-column form
+    // `(scraped, status, redirectDestId)` is fully covering for
+    // `SELECT id` and gives a 5113 ms → 14 ms (~365x) reduction
+    // verified by `scripts/prototype-summary-indexes.mjs`.
+    //
+    // Note the column order: `status` comes second so the `(scraped=1
+    // AND status=-1)` 2-column equality seek lands directly in the
+    // failed-page slice without scanning the 400 k+ healthy rows.
+    // Putting `redirectDestId` last keeps it as a 3rd-level seek
+    // constraint that the planner folds into the slice once the
+    // (scraped, status) pair is fixed.
+    await instance.raw('CREATE INDEX idx_pages_summary_failed ON pages(scraped, status, redirectDestId)');
 }

package/lib/archive/is-inventory-source.d.ts

@@ -0,0 +1,21 @@
+import type { PageSource } from './types.js';
+/**
+ * Predicate that returns `true` when the given page source value belongs to
+ * the inventory chain — i.e. it is one of the `'inventory-*'` variants of
+ * {@link PageSource}.
+ *
+ * Centralises the membership check that decides whether lineage
+ * propagation should fire. Three call sites used to inline
+ * `s === 'inventory-seed' || s === 'inventory-discovered'`, which is both
+ * a DRY violation AND a future-proofing trap: when a new inventory-family
+ * label gets added (e.g. `'inventory-promoted'`), every inlined check has
+ * to be located and updated by hand. Routing through this predicate
+ * keeps the membership rule in one place.
+ *
+ * Returns `false` for `undefined` so callers can pass the raw `source`
+ * column value (which is non-NULL in the DB schema but reads as
+ * `undefined` from a missing row in JS) without a separate null check.
+ * @param source - The source value to test, or `undefined` when no row matched.
+ * @returns `true` if the source is in the inventory chain, `false` otherwise.
+ */
+export declare function isInventorySource(source: PageSource | undefined): boolean;

package/lib/archive/is-inventory-source.js

@@ -0,0 +1,22 @@
+/**
+ * Predicate that returns `true` when the given page source value belongs to
+ * the inventory chain — i.e. it is one of the `'inventory-*'` variants of
+ * {@link PageSource}.
+ *
+ * Centralises the membership check that decides whether lineage
+ * propagation should fire. Three call sites used to inline
+ * `s === 'inventory-seed' || s === 'inventory-discovered'`, which is both
+ * a DRY violation AND a future-proofing trap: when a new inventory-family
+ * label gets added (e.g. `'inventory-promoted'`), every inlined check has
+ * to be located and updated by hand. Routing through this predicate
+ * keeps the membership rule in one place.
+ *
+ * Returns `false` for `undefined` so callers can pass the raw `source`
+ * column value (which is non-NULL in the DB schema but reads as
+ * `undefined` from a missing row in JS) without a separate null check.
+ * @param source - The source value to test, or `undefined` when no row matched.
+ * @returns `true` if the source is in the inventory chain, `false` otherwise.
+ */
+export function isInventorySource(source) {
+    return source === 'inventory-seed' || source === 'inventory-discovered';
+}

package/lib/archive/migrate-inventory-runs.d.ts

@@ -0,0 +1,29 @@
+import type { Knex } from 'knex';
+/**
+ * Adds the `inventory_runs` audit-log table to archives created before
+ * Phase 1 of inventory run tracking shipped.
+ *
+ * `inventory_runs` records one row per successful `--inventory <list>`
+ * invocation, capturing which deploy list was applied when and what
+ * scale it operated at. The motivation is operational: client /
+ * director conversations repeatedly ask "did you apply last month's
+ * list" / "we didn't double-apply, right" — the archive itself had no
+ * trace of inventory passes (`.bak` is unlinked on success), so this
+ * table is the durable provenance record.
+ *
+ * Schema details (column semantics, NULL policy, index) live in
+ * {@link initSchema} — this migration only re-creates the table shape
+ * on legacy archives so the rest of the codebase can treat the table
+ * as always-present once a writer connection has opened the file.
+ *
+ * Idempotent: when the table already exists, the function exits
+ * silently — the `[migrate] inventory_runs table created` stderr line
+ * fires **only** on the first run against a legacy archive, matching
+ * the established pattern of `migrate-page-errors.ts` /
+ * `migrate-crawl-errors.ts`. Operators can rely on the log line as a
+ * stable "first time this archive saw Phase 1 schema" event marker.
+ * Empty archives (no `pages` table) are skipped entirely; the regular
+ * `initSchema` path provisions them at first crawl.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function migrateInventoryRuns(instance: Knex): Promise<void>;

package/lib/archive/migrate-inventory-runs.js

@@ -0,0 +1,52 @@
+/**
+ * Adds the `inventory_runs` audit-log table to archives created before
+ * Phase 1 of inventory run tracking shipped.
+ *
+ * `inventory_runs` records one row per successful `--inventory <list>`
+ * invocation, capturing which deploy list was applied when and what
+ * scale it operated at. The motivation is operational: client /
+ * director conversations repeatedly ask "did you apply last month's
+ * list" / "we didn't double-apply, right" — the archive itself had no
+ * trace of inventory passes (`.bak` is unlinked on success), so this
+ * table is the durable provenance record.
+ *
+ * Schema details (column semantics, NULL policy, index) live in
+ * {@link initSchema} — this migration only re-creates the table shape
+ * on legacy archives so the rest of the codebase can treat the table
+ * as always-present once a writer connection has opened the file.
+ *
+ * Idempotent: when the table already exists, the function exits
+ * silently — the `[migrate] inventory_runs table created` stderr line
+ * fires **only** on the first run against a legacy archive, matching
+ * the established pattern of `migrate-page-errors.ts` /
+ * `migrate-crawl-errors.ts`. Operators can rely on the log line as a
+ * stable "first time this archive saw Phase 1 schema" event marker.
+ * Empty archives (no `pages` table) are skipped entirely; the regular
+ * `initSchema` path provisions them at first crawl.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function migrateInventoryRuns(instance) {
+    const hasTable = await instance.schema.hasTable('inventory_runs');
+    if (hasTable) {
+        return;
+    }
+    const hasPages = await instance.schema.hasTable('pages');
+    if (!hasPages) {
+        // Empty archive; the regular initSchema path will create the table.
+        return;
+    }
+    await instance.schema.createTable('inventory_runs', (t) => {
+        t.increments('id');
+        t.string('ran_at').notNullable();
+        t.string('list_label').nullable();
+        t.string('source_file_sha256', 64).nullable();
+        t.integer('total_lines').nullable();
+        t.integer('new_pages').nullable();
+        t.integer('new_resources').nullable();
+        t.integer('scope_skipped').nullable();
+        t.text('notes').nullable();
+        t.index('ran_at');
+    });
+    // eslint-disable-next-line no-console
+    console.error('[migrate] inventory_runs table created');
+}

package/lib/archive/types.d.ts

@@ -68,6 +68,39 @@ export interface Config extends Required<Pick<ParseURLOptions, 'disableQueries'>
  * the row.
  */
 export type PageSource = 'crawled' | 'inventory-seed' | 'inventory-discovered';
+/**
+ * One row written to the `inventory_runs` audit table on each successful
+ * `--inventory <list>` invocation.
+ *
+ * Schema-mirror interface: every column on `inventory_runs` is represented
+ * here. Only `ran_at` is required — every other field is nullable so the
+ * post-merge raw-SQL backfill path (a one-off `sqlite3 INSERT` for the
+ * initial inventory pass that predated this table) can omit summary
+ * stats it cannot reconstruct.
+ *
+ * The audit log is append-only at Phase 1: there is no UPDATE path, no
+ * UNIQUE constraint on `source_file_sha256`, and no FK to pages /
+ * resources. Phase 2 (`inventory_memberships`) introduces the M:N link
+ * to URLs; Phase 3 (`--refresh`) uses `source_file_sha256` for dedupe.
+ */
+export interface InventoryRunMeta {
+    /** ISO 8601 timestamp at which the run completed (e.g. `'2026-06-21T11:30:00+09:00'`). */
+    ran_at: string;
+    /** Human-readable identifier (e.g. `'prod-2026-06-21'`). `null` when the caller did not supply one. */
+    list_label?: string | null;
+    /** SHA-256 hex digest of the source file. `null` if hashing failed (e.g. file vanished mid-run). */
+    source_file_sha256?: string | null;
+    /** Number of non-empty lines in the input list (= URL count before scope filtering). */
+    total_lines?: number | null;
+    /** Number of new HTML seeds inserted as `pages` rows by this run. */
+    new_pages?: number | null;
+    /** Number of new non-HTML URLs inserted as `resources` rows by this run. */
+    new_resources?: number | null;
+    /** Number of input URLs dropped because they fell outside the archived scope. */
+    scope_skipped?: number | null;
+    /** Free-form text for backfill annotations or operator notes. */
+    notes?: string | null;
+}
 /**
  * Filter type for querying pages from the database.
  *

package/lib/classify-error-kind.d.ts

@@ -0,0 +1,19 @@
+import type { ErrorKind } from './types.js';
+/**
+ * Classify a raw crawler/scraper error message into a coarse {@link ErrorKind}.
+ *
+ * Pure and deterministic: the same message always yields the same kind, which
+ * is why the kind is derived on read rather than persisted — it can be applied
+ * uniformly to freshly captured `crawl_errors`, legacy `error.log` lines, and
+ * `page_errors` alike.
+ * @param message - The raw error message (a single line is sufficient; the
+ *   cause token such as `ENOTFOUND` or `Navigation timeout` lives there).
+ * @returns The matched kind, or `unknown` when no matcher applies.
+ * @example
+ * ```ts
+ * classifyErrorKind('getaddrinfo ENOTFOUND www.example.com'); // 'dns'
+ * classifyErrorKind('gave up after 3 retries — Race 180,000ms'); // 'timeout'
+ * classifyErrorKind('Protocol error (Page.reload): Target closed'); // 'protocol'
+ * ```
+ */
+export declare function classifyErrorKind(message: string): ErrorKind;