@nitpicker/crawler 0.9.0 → 0.12.0

package/lib/archive/cache/resolve-archive-cache-dir.js

@@ -0,0 +1,32 @@
+import path from 'node:path';
+/**
+ * Build the absolute path to a single archive's cache directory.
+ *
+ * The directory name is `<cacheKey>-<safeBasename>`:
+ *
+ * - `cacheKey` carries the freshness signal (size/mtime/ctime) and is
+ *   what actually drives cache hit vs miss.
+ * - `safeBasename` is appended purely so a human running `ls` on the
+ *   cache root can recognise which archive an entry belongs to. It is
+ *   NOT used to disambiguate keys — two archives with identical inode
+ *   metadata (e.g. an identical copy under a different name) intentionally
+ *   share an entry under the first basename that landed there.
+ *
+ * The basename is sanitised: anything outside `[A-Za-z0-9._-]` becomes
+ * `_`. This keeps the path portable across filesystems (no spaces,
+ * unicode normalisation surprises, Windows-reserved chars) and removes
+ * any chance that a crafted archive name could escape the cache root
+ * (e.g. via `..` or path separators), independent of the upstream
+ * `path.basename` call that already drops directory components.
+ * @param cacheRoot - Absolute path returned by `getArchiveCacheRoot()`.
+ * @param cacheKey - The freshness key from `computeArchiveCacheKey()`.
+ * @param archivePath - Absolute path to the source `.nitpicker` file;
+ *   only its basename contributes to the cache entry name.
+ * @returns Absolute path to the per-archive cache directory.
+ */
+export function resolveArchiveCacheDir(cacheRoot, cacheKey, archivePath) {
+    const rawBasename = path.basename(archivePath, path.extname(archivePath));
+    const safeBasename = rawBasename.replaceAll(/[^\w.-]+/g, '_').slice(0, 80);
+    const dirName = safeBasename.length > 0 ? `${cacheKey}-${safeBasename}` : cacheKey;
+    return path.resolve(cacheRoot, dirName);
+}

package/lib/archive/database.d.ts

@@ -1,4 +1,5 @@
-import type { Config, DatabaseOption, DB_Anchor, DB_Page, DB_Redirect, DB_Referrer, DB_Resource, DatabaseEvent, PageFilter } from './types.js';
+import type { JsonLdRow, TagRow } from './meta/types.js';
+import type { Config, DatabaseOption, DB_Anchor, DB_Page, DB_Redirect, DB_Referrer, DB_Resource, DatabaseEvent, InventoryRunMeta, PageFilter, PageSource } from './types.js';
 import type { PageData, Resource } from '../utils/types/types.js';
 import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
 import type { Knex } from 'knex';
@@ -6,10 +7,19 @@ import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-awa
 /**
  * Low-level database abstraction layer for the archive's SQLite database.
  *
- * Manages the `pages`, `anchors`, `images`, `resources`, and `resources-referrers`
- * tables. All public methods that perform database queries use the `@retryable`
- * decorator for automatic retry on transient failures, and `@ErrorEmitter` to
- * propagate errors as events.
+ * Public methods that perform database queries use the `emitErrorAndRetry`
+ * HOF for automatic retry on transient failures combined with error-event
+ * propagation, or `emitError` when retry is not appropriate. The set of
+ * tables this layer manages is
+ * defined by `init-schema.ts` (the source of truth — query that file for
+ * the canonical list).
+ *
+ * **Label sync caveat**: each `emitError` / `emitErrorAndRetry` call passes
+ * the method name as a string literal (e.g. `'Database.getAnchorsOnPage'`).
+ * TypeScript cannot check that the string matches the enclosing method's
+ * real name — the two-way sync is manual. Renaming a method here **must**
+ * update the literal string too, otherwise debug logs and `RetryTimeoutError`
+ * messages will silently report the old name.
  *
  * Use the static {@link Database.connect} factory method to create instances.
  * The constructor is private.
@@ -30,12 +40,6 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * This ensures the database is fully self-contained in `db.sqlite` before archiving.
      */
     checkpoint(): Promise<void>;
-    /**
-     * Clears the HTML snapshot path for a page.
-     * Used to roll back the snapshot reference when the snapshot file write fails.
-     * @param pageId - The database ID of the page whose HTML path should be cleared.
-     */
-    clearHtmlPath(pageId: number): Promise<void>;
     /**
      * Destroys the database connection, releasing all pooled resources.
      */
@@ -62,18 +66,118 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
     getConfig(): Promise<Config>;
     /**
      * Retrieves the current crawling state by listing scraped and pending URLs.
-     * @returns An object with `scraped` (completed URLs) and `pending` (remaining URLs) arrays.
+     *
+     * `scraped` is straightforward: every page row whose `scraped` flag is `1`
+     * — that is, every URL the crawl reached a terminal state on, including
+     * setSkippedPage / setExternalPage / outright setPage success or failure.
+     *
+     * `pending` is intentionally STRICT — not "every `scraped = 0` row".
+     * Three filters apply:
+     *
+     * 1. `scraped = 0` — work still incomplete.
+     * 2. `isExternal = 0` — only in-scope work. External URLs go through a
+     *    HEAD-only path that always lands on `scraped = 1` (either setPage or
+     *    setExternalPage). A row with `isExternal = 1 AND scraped = 0` is
+     *    therefore a data anomaly, and resume / inventory / append have no
+     *    business retrying it on the next session.
+     * 3. `EXISTS (anchor with hrefId = pages.id) OR source != 'crawled'` —
+     *    the row was either discovered as an anchor destination during a
+     *    previous scrape OR was explicitly tagged with a non-default
+     *    source label (`'inventory-seed'`, `'inventory-discovered'`, …).
+     *    Both halves of the OR represent "deliberately enqueued, expected
+     *    to be processed", which is exactly what `resume` should pick up.
+     *
+     *    The orphan filter targets the **predicted-discard leak** in
+     *    `crawler.ts` where `shouldDiscardPredicted` returns true but no
+     *    `emit('skip')` follows. Such placeholders are inserted with the
+     *    DB DEFAULT `source = 'crawled'` (no caller explicitly labels
+     *    them) AND have no anchor referrer (predicted URLs are
+     *    synthesised from pagination patterns, never anchored from a
+     *    rendered page) — both halves of the OR are therefore false and
+     *    the leak is excluded.
+     *
+     *    The `source != 'crawled'` clause specifically saves the
+     *    `--inventory` × `--retry-failed` interaction: an inventory-seed
+     *    URL came from the operator's URL list (no anchor referrer) and
+     *    `resetFailedPages` puts it back at `scraped = 0`. Without this
+     *    clause those legitimate retries would be dropped on resume.
+     *
+     * The defensive shape is on purpose: the data source can drift into
+     * anomalous states under interruption, but the reader must never throw
+     * or feed garbage back into the dealer. A real in-scope URL that was
+     * truly interrupted mid-crawl will always have at least one anchor
+     * referrer (otherwise the dealer would not have queued it), so the
+     * strict filter loses no legitimate pending work.
+     *
+     * Seeds passed directly to `Crawler.start()` are NOT in the strict
+     * pending set when they were never picked by the dealer — they have no
+     * DB row at all in that case (`linkList.add` is purely in-memory until
+     * `setPage` runs). A Ctrl-C between dealer pick and `setPage` likewise
+     * leaves no row to recover. Recovery of un-picked seeds is the
+     * responsibility of the caller (e.g. re-running `--inventory ./list.txt`
+     * with the same URL list).
+     *
+     * The query uses an explicit `p` alias on the `pages` table so the
+     * correlated `EXISTS` subquery can join via `whereRaw('anchors.hrefId =
+     * p.id')`. A future refactor that renames the alias must update both
+     * sites — the raw string in the subquery cannot be grep-resolved
+     * automatically. Read-only / stub viewer connections never call this
+     * method (they do not need to know about pending state), so the EXISTS
+     * shape is safe to use without the `migrate*` guards that other writer
+     * methods carry.
+     * @returns An object with `scraped` (completed URLs) and `pending` (the
+     *   strict set of in-scope, anchor-referenced, unfinished URLs).
      */
     getCrawlingState(): Promise<{
         scraped: string[];
         pending: string[];
     }>;
     /**
-     * Retrieves the HTML snapshot file path for a specific page.
+     * Return the subset of `urls` that already exist in the `pages` table.
+     * Chunked into batches so SQLite's `IN (?, ?, …)` parameter limit
+     * (`SQLITE_MAX_VARIABLE_NUMBER`, default 999) cannot be hit even when the
+     * inventory list contains tens of thousands of URLs.
+     *
+     * Read-only — no transaction, no lock contention with the crawler write
+     * pipeline (callers run this BEFORE the `<archive>.bak` is taken and the
+     * crawl is started).
+     * @param urls - URL strings to probe (already in `withoutHashAndAuth` form).
+     * @returns URLs found in `pages`. Order is not preserved.
+     */
+    getExistingPageUrls(urls: readonly string[]): Promise<string[]>;
+    /**
+     * Return the subset of `urls` that already exist in the `resources` table.
+     * See {@link Database.getExistingPageUrls} — same chunking strategy.
+     * @param urls - URL strings to probe.
+     * @returns URLs found in `resources`.
+     */
+    getExistingResourceUrls(urls: readonly string[]): Promise<string[]>;
+    /**
+     * Reads the HTML snapshot stored as a zstd-compressed BLOB for the given page.
+     *
+     * Joins `page_html_ref` → `page_html_blobs` and decompresses inline. Returns
+     * `null` when the page has no stored body (a non-HTML resource, a redirect
+     * source, a degraded render). Read works identically on read-only / stub
+     * connections — the special-cased "do we have a loose dir vs zip?" branching
+     * the previous file-backed layout required is gone.
+     *
+     * Tables `page_html_ref` and `page_html_blobs` are created by `initSchema`.
+     * Older `.nitpicker` archives that predate this migration must be passed
+     * through `scripts/migrate-to-0.10.mjs` before they can be read.
      * @param pageId - The database ID of the page.
-     * @returns The relative file path to the HTML snapshot, or null if not saved.
+     * @returns The decompressed HTML string, or `null` if no snapshot is stored.
      */
-    getHtmlPathOnPage(pageId: number): Promise<any>;
+    getHtmlOfPageById(pageId: number): Promise<string | null>;
+    /**
+     * Retrieves all `page_jsonld` rows for the given page id, parsed back into
+     * {@link JsonLdRow} shape (with `parsed` deserialised from its JSON column).
+     *
+     * Read-side counterpart to `#insertJsonLd`. Returns rows in insertion order
+     * by `id` so the order observed by `get-page-jsonld` matches the order the
+     * scraper saw them.
+     * @param pageId
+     */
+    getJsonLdOfPage(pageId: number): Promise<JsonLdRow[]>;
     /**
      * Returns the underlying Knex query builder instance for direct SQL access.
      * This enables advanced queries (GROUP BY, HAVING, JOINs) at the database
@@ -101,6 +205,27 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @returns An array of raw {@link DB_Page} rows.
      */
     getPages(filter?: PageFilter, offset?: number, limit?: number): Promise<DB_Page[]>;
+    /**
+     * Look up the `source` column of a single page by its URL key. Used by
+     * the orchestrator's `PageSourceLookup` injection so the Crawler can
+     * resolve a parent page's lineage on `--resume` / `--retry-failed`
+     * sessions, where the in-memory `inventoryMode` is no longer
+     * available but the DB still remembers what label was last persisted.
+     *
+     * Returns `undefined` when the URL has no `pages` row (e.g. a brand-new
+     * URL that has not been seen yet) so the caller can fall through to
+     * its default behaviour without distinguishing "row absent" from "row
+     * present with NULL source" — the schema's `NOT NULL DEFAULT 'crawled'`
+     * makes a NULL value impossible in practice.
+     *
+     * Read-only — no transaction, single PK-equivalent lookup on
+     * `pages.url` (a UNIQUE column), so the cost is constant per call. The
+     * Crawler calls this at most once per page render, NOT per
+     * sub-resource, so the N+1 risk does not apply.
+     * @param url - URL key in `url.withoutHashAndAuth` form.
+     * @returns The recorded `source`, or `undefined` when no row exists.
+     */
+    getPageSourceByUrl(url: string): Promise<PageSource | undefined>;
     /**
      * Retrieves pages along with their related redirect, anchor, and referrer data.
      * Results are ordered by the natural URL sort order. Only non-redirected pages are returned.
@@ -122,6 +247,14 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
     getRedirectsForPages(pageIds: number[]): Promise<DB_Redirect[]>;
     /**
      * Retrieves pages that link to a specific page (incoming links / referrers).
+     *
+     * Incoming links are resolved **through redirects**: an anchor pointing at a
+     * redirect source (e.g. `http://x` that 301s to `https://x`) counts as a
+     * referrer of the redirect's final destination, not of the source. This keeps
+     * backlinks merged on the canonical page instead of splitting them across the
+     * `http`/`https` (or any redirect source/dest) pair. The resolution mirrors
+     * `redirectTable()` — `redirectDestId` is pre-flattened to the final
+     * destination, so `COALESCE(target.redirectDestId, target.id)` is a single hop.
      * @param pageId - The database ID of the target page.
      * @returns An array of referrer records with URL, hash, and text content.
      */
@@ -132,6 +265,22 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @returns An array of page URL strings that reference the resource.
      */
     getReferrersOfResource(id: number): Promise<string[]>;
+    /**
+     * Retrieves a single sub-resource from the `resources` table by its URL.
+     *
+     * Accepts multiple URL candidates because the stored key is the resource's
+     * `href` while callers may only know the hash-stripped form; the first match
+     * wins.
+     *
+     * Deliberately NOT wrapped with `emitError`/`emitErrorAndRetry`: the only caller (the
+     * crawler's resource-reuse hook) has a full fallback (the HEAD pre-flight),
+     * so a read failure here must not surface as a database `error` event —
+     * the orchestrator aborts the whole crawl on that event, which is the
+     * correct reaction to write failures but not to a recoverable read.
+     * @param urls - URL candidates to match against the `url` column.
+     * @returns The raw {@link DB_Resource} row, or `null` if none match.
+     */
+    getResourceByUrl(urls: readonly string[]): Promise<DB_Resource | null>;
     /**
      * Retrieves all sub-resources from the `resources` table.
      * @returns An array of raw {@link DB_Resource} rows.
@@ -142,12 +291,125 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @returns An array of resource URL strings.
      */
     getResourceUrlList(): Promise<any[]>;
+    /**
+     * Counts pages that were scraped as crawl targets (full HTML render).
+     *
+     * Used by the crawler to seed its `pagesScraped` counter on resume so the
+     * progress display reflects all browser-rendered HTML pages across sessions,
+     * not just the current one.
+     *
+     * "HTML page" is guaranteed by `contentType = 'text/html'`, NOT by `isTarget`
+     * alone: `isTarget` means "in-scope crawl target" and is set for in-scope
+     * non-HTML resources too (e.g. a PDF reached via the HEAD pre-flight is
+     * `isTarget = 1`). Counting those would over-report the HTML page total, so
+     * page-ness is asserted at the read layer here rather than by trusting
+     * `isTarget`.
+     * @returns The number of `text/html` rows with `isTarget = 1` and `scraped = 1`.
+     */
+    getScrapedHtmlPageCount(): Promise<number>;
+    /**
+     * Retrieves all `page_tags` rows for the given page id, parsed back into
+     * {@link TagRow} shape (with `categories` and `sources` JSON columns
+     * deserialised).
+     *
+     * Read-side counterpart to `#insertTags`.
+     * @param pageId
+     */
+    getTagsOfPage(pageId: number): Promise<TagRow[]>;
+    /**
+     * Records a crawler-level (`error` channel) failure into `crawl_errors`.
+     *
+     * Unlike {@link insertPageError} this is not tied to a scraped page: `url`
+     * may be an external link that never became a page row, or `null` for a
+     * process-level error. The cause is intentionally not stored — it is derived
+     * on read so that older archives (which only have `error.log`) and freshly
+     * captured rows classify identically.
+     * @param url - The URL the error is about, or `null` for a process-level error.
+     * @param message - The error message (one line is enough for classification).
+     * @param isExternal - Whether the URL is external to the crawl scope.
+     */
+    insertCrawlError(url: string | null, message: string, isExternal?: boolean): Promise<void>;
+    /**
+     * Pre-insert inventory non-HTML URLs into `resources` as placeholder rows
+     * with `source = 'inventory-seed'` and all metadata columns NULL — the
+     * non-HTML counterpart of {@link Database.insertInventorySeeds}. Used by
+     * `CrawlerOrchestrator.inventory` so the ingestion phase commits all of
+     * its non-HTML URLs in one chunked round-trip per 500 instead of N
+     * sequential `insertResource` awaits. On a 50k-URL inventory list the
+     * old per-URL loop spent minutes inside the `.bak`-protected window;
+     * the bulk path finishes in seconds.
+     *
+     * Idempotent: `onConflict('url').ignore()` leaves existing rows untouched
+     * (the orchestrator's `getExistingResourceUrls` filter is what keeps a
+     * crawled-lineage `resources` row from being downgraded to the
+     * inventory label here).
+     *
+     * Chunked at 500 to stay well under SQLite's `SQLITE_MAX_VARIABLE_NUMBER`
+     * (default 999) — every row binds the URL plus the `responseHeaders`
+     * JSON null, so the per-chunk bound budget is well within limits.
+     * @param urls - URL strings (already in `withoutHashAndAuth` form).
+     */
+    insertInventoryResources(urls: readonly string[]): Promise<void>;
+    /**
+     * Pre-insert inventory HTML seeds into `pages` as `scraped = 0`,
+     * `source = 'inventory-seed'` placeholders so the URL's existence in the
+     * archive is **durable before the scrape phase starts**.
+     *
+     * Why this is the linchpin of `--inventory` Ctrl+C tolerance: HTML seeds
+     * used to live only in the Crawler's in-memory `LinkList` until the
+     * dealer eventually called `setPage`. A Ctrl+C / crash before that point
+     * lost the seed without trace, and `--resume` could not recover it
+     * because `getCrawlingState`'s strict pending set requires a `pages` row.
+     * Pre-inserting fills exactly that gap: the strict pending set picks
+     * these rows up via its `OR p.source != 'crawled'` clause, so
+     * `--resume` after an interrupted inventory pass picks every seed back
+     * up. See {@link Database.getCrawlingState} for the strict-set rationale.
+     *
+     * Idempotent: `onConflict('url').ignore()` keeps existing rows intact.
+     * The {@link Database.#getIdByUrl} crawled-wins downgrade still fires
+     * later when a crawled-lineage anchor reaches one of these seeds —
+     * that's the right behaviour (a seed that turned out to be reachable
+     * is not an orphan and should not retain the inventory label).
+     *
+     * Chunked into 500-URL batches so SQLite's bound-parameter limit
+     * (`SQLITE_MAX_VARIABLE_NUMBER`, default 999) cannot be hit even on a
+     * tens-of-thousands inventory list.
+     *
+     * Called by {@link CrawlerOrchestrator.inventory} during the
+     * `.bak`-protected ingestion phase, so any failure here aborts the run
+     * and restores from backup — the operator reruns from scratch.
+     * @param urls - URL strings already in `withoutHashAndAuth` form.
+     */
+    insertInventorySeeds(urls: readonly string[]): Promise<void>;
+    /**
+     * Records a partial scrape failure against the page identified by `url`.
+     *
+     * The page row is resolved (or inserted as a stub) via
+     * {@link Database.#getIdByUrl} so the error can be recorded even before
+     * `setPage` has run — useful when the failure fires during scraping
+     * (e.g. mid-`scrapeStart`) and the orchestrator enqueues this write
+     * before the success write for the same URL.
+     *
+     * A single page can have multiple `page_errors` rows (e.g. both
+     * `desktop-compact` and `mobile-small` viewports failing).
+     * @param url - URL of the page being scraped.
+     * @param phase - Scrape phase name (typically `'retryExhausted'`).
+     * @param message - Human-readable failure message.
+     * @param isExternal - Whether the URL is external. Defaults to `false`.
+     */
+    insertPageError(url: string, phase: string, message: string, isExternal?: boolean): Promise<void>;
     /**
      * Inserts a sub-resource into the `resources` table.
      * Ignores duplicate URLs (uses `ON CONFLICT IGNORE`).
+     *
+     * The `source` provenance label is written ONLY on insert; an
+     * `ON CONFLICT IGNORE` collision leaves an existing row's source untouched
+     * (this is what makes a second `crawl --inventory` non-destructive — see
+     * the inventory plan).
      * @param resource - The resource data to insert.
+     * @param source - Provenance label for new rows. `undefined` leaves the DB DEFAULT (`'crawled'`).
      */
-    insertResource(resource: Resource): Promise<void>;
+    insertResource(resource: Resource, source?: PageSource): Promise<void>;
     /**
      * Inserts a referrer relationship between a resource and a page into the
      * `resources-referrers` table. Silently skips if the resource is not found.
@@ -155,6 +417,72 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @param pageUrl - The URL of the page that references the resource.
      */
     insertResourceReferrers(src: string, pageUrl: string): Promise<void>;
+    /**
+     * Hostnames whose `crawl_errors` history is consistently DNS failures and
+     * for which no recent 2xx-3xx page or resource is recorded — i.e. hosts
+     * the previous crawl already proved unreachable. Returned in lower-cased
+     * form. Used by `CrawlerOrchestrator.#preloadDnsBurnedHostCache` so the
+     * next session short-circuits HEAD pre-flight on these hosts.
+     *
+     * Implementation: a coarse `LIKE` filter over `crawl_errors.message`
+     * narrows the row set, then `classifyErrorKind` confirms `'dns'` in JS
+     * (the regex is the single truth source — DB-side filters never narrow
+     * it). Exclusion bags are built from a single `pages` and a single
+     * `resources` scan: any host with a 2xx-3xx page, a 2xx-3xx resource, or
+     * a `pages.lastCrawledAt` newer than its latest DNS error is dropped
+     * (the host probably recovered between the failure and the last crawl).
+     *
+     * Returns `[]` on legacy archives that pre-date the `crawl_errors`
+     * table — the `hasTable` guard keeps the call non-destructive.
+     * @returns Lower-cased hostnames safe to short-circuit.
+     */
+    listDnsBurnedHostCandidates(): Promise<string[]>;
+    /**
+     * Appends one row to the `inventory_runs` audit log.
+     *
+     * Called by {@link CrawlerOrchestrator.inventory} on every successful
+     * `--inventory <list>` invocation so the archive carries a durable
+     * record of which deploy list was applied when and at what scale —
+     * the operational question "did we apply last month's list" the
+     * archive itself can answer without consulting external bookkeeping.
+     *
+     * Append-only at Phase 1. There is intentionally no UPDATE path and
+     * no UNIQUE constraint on `source_file_sha256`; two applies of the
+     * same list each get their own row, and `Phase 3 --refresh` is where
+     * dedupe / pre-flight against the hash will land. Field-level NULL
+     * semantics live on {@link InventoryRunMeta}.
+     * @param meta - The run metadata to record. Only `ran_at` is required.
+     * @returns The autoincremented `id` of the newly-inserted row.
+     */
+    recordInventoryRun(meta: InventoryRunMeta): Promise<number>;
+    /**
+     * Records a redirect edge (source → destination) **without** re-storing the
+     * destination's content.
+     *
+     * The crawler renders a many-to-one redirect destination exactly once. For
+     * every subsequent source URL that redirects to that already-rendered
+     * destination, it calls this instead of {@link updatePage} (#73). Routing a
+     * content-less HEAD result through `updatePage` would funnel it into
+     * `#insertPage` and overwrite the destination's good title / meta with empty
+     * values, so the dedicated edge-only path is required.
+     *
+     * The destination row is resolved (created on demand if a concurrent in-flight
+     * render has not committed it yet) so the edge always points at a valid id;
+     * the single render fills in the destination's content under that same id.
+     * The destination's existing anchors / images are never touched here.
+     * @param page - HEAD-resolved page data carrying the redirect chain. Its
+     *   `anchorList` / `imageList` are ignored (a redirect source owns no content).
+     * @param source - Inventory provenance forwarded by the orchestrator
+     *   (`Archive.setRedirect` → here) for the redirect-edge fast path. Used
+     *   as the fallback when the originating URL's row does NOT yet exist in
+     *   the archive (`#73` convergence on first sight, js-redirect rescue
+     *   before any prior write). When the originating row already exists
+     *   (e.g. anchor-lineage INSERT from a prior pass), its stored `source`
+     *   takes precedence so transitive lineage is preserved across resume /
+     *   retry-failed sessions. `undefined` keeps the DB DEFAULT `'crawled'`
+     *   on a brand-new destination row.
+     */
+    recordRedirect(page: PageData, source?: PageSource): Promise<void>;
     /**
      * Promote previously-external pages whose URL falls under any of the new scope
      * entries back to a "needs scraping" state so that the next crawl picks them up
@@ -174,6 +502,48 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @returns The URLs of the pages that were promoted.
      */
     repromoteExternalPages(scopes: ReadonlyMap<string, readonly ExURL[]>, options?: ParseURLOptions): Promise<string[]>;
+    /**
+     * Reset previously-attempted pages that ended in a recoverable failure so a
+     * follow-up crawl can re-fetch them from scratch.
+     *
+     * A page qualifies as a recoverable failure when it was already scraped
+     * (`scraped = 1`), is not a redirect source (`redirectDestId IS NULL`), was
+     * not intentionally skipped (`isSkipped` is not `1`), and one of the
+     * following holds:
+     *
+     * - `status = -1` — the sentinel a hard scrape failure (network error,
+     *   timeout, browser crash) is recorded with (see `handle-scrape-error.ts`);
+     * - `status IS NULL` — no status was ever stored for the row;
+     * - `contentType IS NULL` — the content type could not be determined;
+     * - `status` is in the `5xx` range — a (frequently transient) server error.
+     *
+     * Definitive `4xx` responses are intentionally excluded: re-fetching a 404
+     * almost always yields the same answer.
+     *
+     * A second exclusion runs in JS after the SQL candidate scan: any page whose
+     * latest recorded `page_errors` / `crawl_errors` message classifies into a
+     * permanent {@link PERMANENT_ERROR_KINDS} kind (dns / tls / client-blocked /
+     * parse-error / connection-refused) is left as-is rather than reset to
+     * pending. Without this filter, `--retry-failed` never converges: NXDOMAIN
+     * hosts, expired-cert hosts, and `ERR_BLOCKED_BY_CLIENT` ad pixels would be
+     * reset every iteration, re-attempted, fail identically, and rejoin the
+     * candidate pool for the next iteration. The exclusion keeps the retry
+     * target shrinking across `--retry-failed` passes by leaving deterministic
+     * dead-ends alone.
+     *
+     * Matching rows — internal and external alike — are demoted back to pending
+     * (`scraped = 0`) and have their stale scrape metadata cleared. The page row
+     * itself is kept (id preserved) so existing `anchors.hrefId` referrers stay
+     * valid, and `isExternal` is left untouched so the next pass re-classifies
+     * each page from the crawl scope. Related `anchors`, `images`,
+     * `resources-referrers`, and `page_errors` rows are deleted so the re-scrape
+     * can re-insert fresh data without duplicates.
+     *
+     * SELECT and UPDATE/DELETE statements are chunked to stay below SQLite's
+     * `SQLITE_LIMIT_VARIABLE_NUMBER`.
+     * @returns The URLs of the pages that were reset to pending.
+     */
+    resetFailedPages(): Promise<string[]>;
     /**
      * Stores the crawl configuration in the `info` table.
      * Only fields in {@link INFO_COLUMN_ALLOWLIST} are forwarded — any extra
@@ -212,27 +582,42 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
     updateConfig(patch: Partial<Config>): Promise<void>;
     /**
      * Inserts or updates a crawled page in the database, including its redirect chain,
-     * anchors, and images. Optionally creates an HTML snapshot file path entry.
+     * anchors, images, and (when `writeHtml`) its compressed HTML snapshot BLOB.
      *
      * Self-redirects (where the source URL equals the destination URL after normalization)
      * are skipped to avoid marking a page as redirected to itself — a situation caused by
      * authentication challenges (e.g. Basic Auth 302) that would otherwise exclude the page
      * from reports via the `whereNull('redirectDestId')` filter.
      * @param page - The page data to store.
-     * @param snapshotDir - The directory for saving HTML snapshots, or null to skip snapshots.
+     * @param writeHtml - When `true`, this call is allowed to insert (or clear)
+     *   the page's HTML blob. `setExternalPage` passes `false` because external
+     *   metadata-only scrapes never carry HTML and must not perturb an already
+     *   stored body.
      * @param isTarget - Whether this page is a crawl target.
-     * @returns An object with the optional `html` snapshot file path and the page's database `pageId`.
+     * @param source - Provenance label written ONLY when the row is freshly
+     *   inserted. Existing rows keep their original `source` (this is why a
+     *   second `crawl --inventory` does not "demote" an `'inventory-seed'` row
+     *   that was discovered earlier).
+     * @returns The database `pageId` of the inserted/updated row.
      */
-    updatePage(page: PageData, snapshotDir: string | null, isTarget: boolean): Promise<{
-        html?: string | undefined;
-        pageId: number;
-    }>;
+    updatePage(page: PageData, writeHtml: boolean, isTarget: boolean, source?: PageSource): Promise<number>;
     /**
      * Creates and initializes a new Database instance.
-     * Creates the parent directory for the database file if needed,
-     * establishes the connection, and initializes tables if they do not exist.
-     * @param options - Database connection options (working directory + SQLite file path).
+     *
+     * **Writer mode (default)**: creates the parent directory for the
+     * database file if needed, establishes the connection, and initializes
+     * the schema + migrations.
+     *
+     * **Read-only mode** (`options.readOnly`): refuses to resurrect a
+     * missing parent directory or db file — throws if either is absent at
+     * the time of the call. Skips schema init and migrations entirely so
+     * the user's tmpDir is never modified. Required by viewer / MCP
+     * stub-mode opens, where a TOCTOU window between classification and
+     * `connect()` could otherwise leave behind a phantom empty tmpDir.
+     * @param options - Database connection options.
      * @returns A fully initialized Database instance.
+     * @throws {Error} In read-only mode, if the parent directory or db
+     *   file does not exist when `connect()` runs.
      */
     static connect(options: DatabaseOption): Promise<Database>;
 }