@nitpicker/crawler 0.9.0 → 0.12.0

package/lib/crawler-orchestrator.js

@@ -6,10 +6,18 @@ import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-awa
 import pkg from '../package.json' with { type: 'json' };
 import Archive from './archive/archive.js';
 import { clearDestinationCache } from './crawler/clear-destination-cache.js';
+import { clearDnsBurnedHostCache } from './crawler/clear-dns-burned-host-cache.js';
 import Crawler from './crawler/crawler.js';
+import { dnsBurnedHostCache } from './crawler/dns-burned-host-cache.js';
+import { dnsBurnedHostShortCircuitCounter } from './crawler/dns-burned-host-short-circuit-counter.js';
+import { findScopeEntry } from './crawler/find-scope-entry.js';
+import { isLikelyHtmlUrl } from './crawler/is-likely-html-url.js';
+import { PreloadShortCircuitError } from './crawler/preload-short-circuit-error.js';
+import { protocolAgnosticKey } from './crawler/protocol-agnostic-key.js';
 import { crawlerLog, log } from './debug.js';
 import { normalizeToArray } from './normalize-to-array.js';
 import { resolveOutputPath } from './resolve-output-path.js';
+import { resourceRowToLookupResult } from './resource-row-to-lookup-result.js';
 import { cleanObject } from './utils/object/clean-object.js';
 import { WriteQueue } from './write-queue.js';
 /**
@@ -53,6 +61,8 @@ export class CrawlerOrchestrator extends EventEmitter {
     #crawler;
     /** Whether the crawl was started from a pre-defined URL list (non-recursive mode). */
     #fromList;
+    /** Serializes archive writes from crawler event handlers (FIFO). */
+    #writeQueue = new WriteQueue();
     /**
      * The underlying archive instance used for storing crawl results.
      */
@@ -95,6 +105,34 @@ export class CrawlerOrchestrator extends EventEmitter {
             verbose: options?.verbose ?? false,
             userAgent: options?.userAgent || defaultUserAgent,
             ignoreRobots: options?.ignoreRobots ?? false,
+            // Let the crawler reuse sub-resource data captured during page
+            // rendering instead of issuing a redundant HEAD pre-flight.
+            lookupResource: async (urls) => {
+                // Fast path: read directly — the row is usually flushed long
+                // before the queued URL is dequeued, and a direct read does not
+                // block behind pending writes.
+                const direct = await this.#archive.getResourceByUrl(urls);
+                if (direct) {
+                    return resourceRowToLookupResult(direct);
+                }
+                // A miss may be an insert still queued — re-read serialized
+                // behind the write queue so hit/miss is deterministic.
+                const row = await this.#writeQueue.enqueue(() => this.#archive.getResourceByUrl(urls));
+                return row ? resourceRowToLookupResult(row) : null;
+            },
+            // Let the crawler propagate the parent's source lineage to
+            // sub-resources on `--resume` / `--retry-failed` sessions, where
+            // `inventoryMode` is not in memory but the DB still remembers
+            // the parent's `source`. Without this, sub-resources captured
+            // during a re-render of an inventory-labelled page would fall
+            // back to the DB DEFAULT `'crawled'` and lose their
+            // `'inventory-discovered'` provenance.
+            lookupPageSource: async (url) => this.#archive.getPageSourceByUrl(url),
+            // Inventory mode is opted into by `CrawlerOrchestrator.inventory`
+            // (see T3); the default crawl path stays in normal mode so new
+            // rows continue to land in pages/resources with the DB DEFAULT
+            // `'crawled'` provenance label.
+            inventoryMode: options?.inventoryMode ?? null,
         });
     }
     /**
@@ -113,30 +151,44 @@ export class CrawlerOrchestrator extends EventEmitter {
      * Sets up event listeners on the crawler, starts crawling, and resolves
      * when the crawl completes. Discovered pages, external pages, skipped pages,
      * and resources are forwarded to the archive for storage.
-     * @param list - The list of parsed URLs to crawl. The first URL is used as the root.
+     * @param list - The list of parsed URLs to crawl. May be empty when a resumed
+     *   session already has pending pages queued (for example `--retry-failed`).
+     * @param opts - Optional crawl overrides.
+     * @param opts.recursive - Whether discovered URLs are followed. Defaults to
+     *   `!fromList` (recursive unless the archive was created from a URL list), so
+     *   existing callers keep their behaviour; the retry flow passes it explicitly.
      * @returns A promise that resolves when crawling is complete.
-     * @throws {Error} If the URL list is empty.
      */
-    async crawling(list) {
-        const root = list[0];
-        if (!root) {
-            throw new Error('URL is empty');
-        }
-        const writeQueue = new WriteQueue();
+    async crawling(list, opts) {
+        const writeQueue = this.#writeQueue;
         return new Promise((resolve, reject) => {
             this.#crawler.on('error', (error) => {
+                if (error.error instanceof PreloadShortCircuitError) {
+                    // DNS-burned host short-circuit: the underlying cause already
+                    // lives in `crawl_errors` from the original DNS failure.
+                    // Writing it again on every subsequent URL would amplify the
+                    // row count on each `--retry-failed` re-run and could even
+                    // inflate the preload selection on the next open. Drop it
+                    // here; `pages.status = -1` still gets set via the normal
+                    // scrape-error path (handleScrapeError → addPageError) so the
+                    // page record itself is unchanged.
+                    crawlerLog('Skipping addError for preload short-circuit: %s', error.url);
+                    return;
+                }
                 crawlerLog('On error: %O', error);
-                void writeQueue.enqueue(() => this.#archive.addError(error));
+                writeQueue
+                    .enqueue(() => this.#archive.addError(error))
+                    .catch((writeError) => reject(writeError));
                 void this.emit('error', error);
             });
-            this.#crawler.on('page', ({ result }) => {
+            this.#crawler.on('page', ({ result, source }) => {
                 writeQueue
-                    .enqueue(() => this.#archive.setPage(result))
+                    .enqueue(() => this.#archive.setPage(result, source))
                     .catch((error) => reject(error));
             });
-            this.#crawler.on('externalPage', ({ result }) => {
+            this.#crawler.on('externalPage', ({ result, source }) => {
                 writeQueue
-                    .enqueue(() => this.#archive.setExternalPage(result))
+                    .enqueue(() => this.#archive.setExternalPage(result, source))
                     .catch((error) => reject(error));
             });
             this.#crawler.on('skip', ({ url, reason, isExternal }) => {
@@ -144,9 +196,20 @@ export class CrawlerOrchestrator extends EventEmitter {
                     .enqueue(() => this.#archive.setSkippedPage(url, reason, isExternal))
                     .catch((error) => reject(error));
             });
-            this.#crawler.on('response', ({ resource }) => {
+            this.#crawler.on('pageError', ({ url, phase, message, isExternal }) => {
+                writeQueue
+                    .enqueue(() => this.#archive.addPageError(url, phase, message, isExternal))
+                    .catch((error) => reject(error));
+            });
+            this.#crawler.on('redirect', ({ result, source }) => {
+                writeQueue
+                    .enqueue(() => this.#archive.setRedirect(result, source))
+                    .catch((error) => reject(error));
+                void this.emit('redirect', { result });
+            });
+            this.#crawler.on('response', ({ resource, source }) => {
                 writeQueue
-                    .enqueue(() => this.#archive.setResources(resource))
+                    .enqueue(() => this.#archive.setResources(resource, source))
                     .catch((error) => reject(error));
             });
             this.#crawler.on('responseReferrers', (resource) => {
@@ -160,7 +223,7 @@ export class CrawlerOrchestrator extends EventEmitter {
                     .then(() => resolve())
                     .catch((error) => reject(error));
             });
-            this.#crawler.start(list, { recursive: !this.#fromList });
+            this.#crawler.start(list, { recursive: opts?.recursive ?? !this.#fromList });
         });
     }
     /**
@@ -271,7 +334,7 @@ export class CrawlerOrchestrator extends EventEmitter {
         log('Config %O', config);
         await orchestrator.crawling(list);
         log('Crawling completed');
-        clearDestinationCache();
+        CrawlerOrchestrator.#finalizeCrawlSession();
         log('Set order natural URL sort');
         await archive.setUrlOrder();
         log('Sorting done');
@@ -348,7 +411,8 @@ export class CrawlerOrchestrator extends EventEmitter {
                 });
                 const { scraped, pending } = await archive.getCrawlingState();
                 const resources = await archive.getResourceUrlList();
-                orchestrator.#crawler.resume(pending, scraped, resources);
+                const pagesScrapedOffset = await archive.getScrapedHtmlPageCount();
+                orchestrator.#crawler.resume(pending, scraped, resources, pagesScrapedOffset);
                 if (initializedCallback) {
                     await initializedCallback(orchestrator, mergedConfig);
                 }
@@ -356,8 +420,9 @@ export class CrawlerOrchestrator extends EventEmitter {
                 log('Archive %s', absFilePath);
                 log('New roots %O', newRoots);
                 log('Merged roots %O', mergedRoots);
+                await CrawlerOrchestrator.#preloadDnsBurnedHostCache(archive);
                 await orchestrator.crawling(newParsed);
-                clearDestinationCache();
+                CrawlerOrchestrator.#finalizeCrawlSession();
                 await archive.setUrlOrder();
                 await ignoreEnoent(unlinkFile(backupPath));
                 return orchestrator;
@@ -381,6 +446,454 @@ export class CrawlerOrchestrator extends EventEmitter {
             throw error;
         }
     }
+    /**
+     * Inventory mode: cross-reference a user-supplied URL list against an
+     * existing `.nitpicker` archive and import ONLY the URLs that are not yet
+     * tracked there. Designed to surface "orphan" landing pages that link
+     * graph traversal could not reach, and "unused" server-side files that
+     * no crawled page references — both of which the
+     * `listIsolatedPages` / `listUnusedResources` queries can then list.
+     *
+     * Flow:
+     *
+     * 1. Open the archive (writer mode, takes the archive lock).
+     * 2. Reject list-mode archives — they hold metadata-only rows that
+     *    inventory has no business touching.
+     * 3. Reject archives with unfinished `pending` URLs — those would inherit
+     *    the inventory `source` label by mistake. Operator must resume /
+     *    retry-failed first.
+     * 4. Parse the URL list. Anything outside the archived scope is warned
+     *    and skipped (inventory is per-server by design).
+     * 5. Subtract URLs that already exist in `pages` or `resources` so the
+     *    second (and N-th) inventory pass is a no-op for known rows — keeps
+     *    `'inventory-seed'` rows from being silently demoted.
+     * 6. Make `<archive>.bak`. Anything thrown beyond this point restores
+     *    from the backup.
+     * 7. HEAD-probe each novel URL. Responses classified as HTML are queued
+     *    as Crawler seeds (`'inventory-seed'`); everything else is recorded
+     *    in `resources` directly as `'inventory-seed'` (no browser launch).
+     * 8. If any HTML seeds exist, start a Crawler with
+     *    `inventoryMode = { seedUrls }` so the rendered page and every newly
+     *    discovered downstream link is labelled correctly. `resume` is fed
+     *    the existing `scraped` / `resources` sets so links into already-
+     *    crawled pages stop at the seen-gate without re-rendering.
+     * 9. Drop the backup on success; restore it on any throw.
+     *
+     * Mutually exclusive with `--append` / `--retry-failed` / `--resume` /
+     * `--diff` / `--list` / `--list-file` / `--single` / `--output` — the
+     * CLI dispatch enforces this; this method assumes the caller honoured
+     * the contract.
+     * @param archivePath - Absolute or cwd-relative path to the `.nitpicker` archive.
+     * @param inventoryUrls - Pre-read URL list (one URL per element).
+     * @param options - Optional config overrides — most callers leave this blank and let the archived config flow through.
+     * @param initializedCallback - Hook invoked once the orchestrator is constructed but before `crawling` runs (the CLI uses it to attach progress reporting).
+     * @param sourceFileSha256 - **Pre-computed** SHA-256 hex digest of the
+     *   source URL list. The orchestrator deliberately does NOT receive
+     *   the file path: the path is privacy-sensitive (leaks user-home /
+     *   OS structure when archives are shared) and we want it lifted off
+     *   this boundary so no future log line / breadcrumb / error message
+     *   inside the orchestrator can accidentally re-leak it. The CLI
+     *   computes the digest via `computeFileSha256(resolvedListFile)`
+     *   and passes it through here. Pass `null` for programmatic
+     *   callers that built `inventoryUrls` in-memory; the audit row's
+     *   `source_file_sha256` column will be `NULL`.
+     * @returns The orchestrator instance after a successful inventory pass.
+     * @throws {Error} When `inventoryUrls` is empty, the archive is in list mode, or pending URLs from a previous crawl remain unresolved.
+     */
+    static async inventory(archivePath, inventoryUrls, options, initializedCallback, sourceFileSha256 = null) {
+        if (inventoryUrls.length === 0) {
+            throw new Error('inventory: URL list is empty');
+        }
+        const cwd = options?.cwd ?? process.cwd();
+        const absFilePath = path.isAbsolute(archivePath)
+            ? archivePath
+            : path.resolve(cwd, archivePath);
+        const archive = await Archive.open({ filePath: absFilePath, cwd });
+        try {
+            const archived = await archive.getConfig();
+            if (archived.fromList) {
+                throw new Error('Cannot run inventory on a list-mode archive: this archive was created with --list/--list-file and contains metadata-only pages. Create a fresh archive instead.');
+            }
+            const { pending } = await archive.getCrawlingState();
+            if (pending.length > 0) {
+                // `getCrawlingState` returns the STRICT pending set — in-scope,
+                // anchor-referenced, `scraped=0` rows. Predicted-discard leaks
+                // and external anomalies are filtered out at the reader, so a
+                // non-empty pending here means the previous session genuinely
+                // stopped with interrupted in-scope work. The original hard
+                // rejection blocked legitimate inventory runs in practice
+                // because leak rows polluted the count; with the strict
+                // reader those false positives are gone, so a warning is
+                // enough — the inventory pass continues and the crawled-wins
+                // source priority keeps stale labels stable even if some of
+                // the strict-pending rows happen to land on inventory seeds.
+                console.warn(`inventory: archive has ${pending.length} pending URLs from a previous crawl. Proceeding — crawled-wins priority keeps their labels stable. Consider \`--resume\` first if you want the prior work finalized.`);
+            }
+            // Parse + scope-classify the candidate URLs. sortUrl drops
+            // unparseable strings; findScopeEntry separates in-scope from
+            // out-of-scope.
+            const parsedAll = sortUrl(inventoryUrls, archived);
+            const scopeMap = new Map();
+            for (const raw of archived.roots) {
+                const parsed = parseUrl(raw, archived);
+                if (!parsed)
+                    continue;
+                const existing = scopeMap.get(parsed.hostname) ?? [];
+                scopeMap.set(parsed.hostname, [...existing, parsed]);
+            }
+            const inScope = [];
+            let outOfScope = 0;
+            for (const url of parsedAll) {
+                if (findScopeEntry(url, scopeMap, archived) === null) {
+                    outOfScope++;
+                }
+                else {
+                    inScope.push(url);
+                }
+            }
+            if (outOfScope > 0) {
+                log('[inventory] %d URL(s) skipped (outside archived scope: %O)', outOfScope, archived.roots);
+            }
+            // Drop URLs that are already represented in the archive (either
+            // as pages or resources). Comparison key is `withoutHashAndAuth`
+            // to mirror what `#getIdByUrl` / `insertResource` actually store.
+            // Two independent reads — Promise.all halves the wait on large
+            // archives where each `WHERE url IN (?)` chunk costs real I/O.
+            const candidateUrls = inScope.map((u) => u.withoutHashAndAuth);
+            const [existingPageUrlList, existingResourceUrlList] = await Promise.all([
+                archive.getExistingPageUrls(candidateUrls),
+                archive.getExistingResourceUrls(candidateUrls),
+            ]);
+            const existingPageUrls = new Set(existingPageUrlList);
+            const existingResourceUrls = new Set(existingResourceUrlList);
+            const novelUrls = inScope.filter((u) => {
+                const key = u.withoutHashAndAuth;
+                return !existingPageUrls.has(key) && !existingResourceUrls.has(key);
+            });
+            const knownCount = existingPageUrls.size + existingResourceUrls.size;
+            log('[inventory] %d in-scope, %d already in archive, %d new', inScope.length, knownCount, novelUrls.length);
+            if (novelUrls.length === 0) {
+                // Nothing to do — release the archive cleanly without taking a
+                // backup. The orchestrator returned here is empty; the caller
+                // should only invoke `close` on it.
+                const noopConfig = {
+                    ...archived,
+                    ...cleanObject(options),
+                };
+                const orchestrator = new CrawlerOrchestrator(archive, noopConfig);
+                if (initializedCallback) {
+                    await initializedCallback(orchestrator, noopConfig);
+                }
+                return orchestrator;
+            }
+            const backupPath = absFilePath + '.bak';
+            await copyFile(absFilePath, backupPath);
+            // Ingestion (pre-insert + audit) is `.bak`-protected — a failure
+            // there restores the archive and the operator reruns. Once
+            // ingestion completes and the `.bak` is released, the scrape
+            // phase runs without `.bak` protection: a Ctrl+C / crash leaves
+            // the pre-inserted `inventory-seed` rows in `pages` so
+            // `crawl --resume` recovers them via the strict-pending set
+            // (see {@link Database.getCrawlingState}'s `OR p.source != 'crawled'`
+            // clause). This flag steers the catch below.
+            let ingestionComplete = false;
+            try {
+                // Classify novel URLs by URL-extension heuristic (no I/O).
+                // Source file lists come from `ls` on the doc-root, so the
+                // extension reflects the real file type — a HEAD pre-flight
+                // here would be pure wasted I/O. Edge cases:
+                //
+                // - `.html` returning 404 / 200: the normal crawler HEAD/GET
+                //   path absorbs this because every HTML-classified URL is
+                //   fed through the dealer and gets its real HEAD/GET there.
+                //
+                // - Extensionless API endpoints (e.g. `/api/foo`) that the
+                //   server returns as `text/html`: `isLikelyHtmlUrl` accepts
+                //   them as HTML so the dealer's render path runs — the
+                //   real content-type wins downstream.
+                //
+                // - `.aspx` / `.do` / `.jsp` / other server-handler
+                //   extensions that the heuristic does NOT recognise as
+                //   HTML: these are classified as non-HTML here, recorded
+                //   as `resources` rows with all-null metadata, and never
+                //   get a HEAD/GET probe. The accepted trade-off for
+                //   `--inventory`'s "list of static-looking server files"
+                //   contract; sites that mix server-handlers into the
+                //   inventory list will need a follow-up `--retry-failed`
+                //   pass (or a re-`--inventory` with the corrected list)
+                //   to populate metadata.
+                //
+                // non-HTML rows are recorded with null status/content-type
+                // which is sufficient for `listUnusedResources` (referrer
+                // count = 0) but means downstream consumers must treat
+                // null as "not probed" rather than "failed".
+                const rawHtmlSeeds = [];
+                const nonHtmlSeeds = [];
+                for (const url of novelUrls) {
+                    if (isLikelyHtmlUrl(url)) {
+                        rawHtmlSeeds.push(url);
+                    }
+                    else {
+                        nonHtmlSeeds.push(url);
+                    }
+                }
+                // Dedup HTML seeds by `protocolAgnosticKey` so an inventory
+                // list that mixes `http://` and `https://` for the same
+                // origin does not produce two `pages` rows that the dealer
+                // later collapses to one — the loser would otherwise stay
+                // `scraped=0, source='inventory-seed'` forever and look like
+                // a real recovery candidate on `--resume`. `getExistingPageUrls`
+                // keys on the full URL (with protocol), so it cannot catch
+                // the cross-scheme duplicate; this is the dedup boundary.
+                const seenKeys = new Set();
+                const htmlSeeds = [];
+                for (const url of rawHtmlSeeds) {
+                    const key = protocolAgnosticKey(url.withoutHashAndAuth);
+                    if (seenKeys.has(key)) {
+                        continue;
+                    }
+                    seenKeys.add(key);
+                    htmlSeeds.push(url);
+                }
+                // Bulk-record non-HTML novel URLs in `resources` as
+                // `source='inventory-seed'` placeholders. The previous
+                // per-URL `await setResources(...)` loop spent minutes
+                // inside the `.bak`-protected window on large inventory
+                // lists; the chunked bulk path collapses N round-trips
+                // to N/500.
+                await archive.insertInventoryResources(nonHtmlSeeds);
+                // Pre-insert HTML seeds as `scraped = 0`,
+                // `source = 'inventory-seed'` placeholders *before* the
+                // scrape phase, so a Ctrl+C between here and `setPage`
+                // no longer loses the URL. The strict-pending set picks
+                // these rows up on the next `--resume` via the
+                // `OR p.source != 'crawled'` clause.
+                await archive.insertInventorySeeds(htmlSeeds);
+                log('[inventory] %d HTML seed(s), %d non-HTML resource(s) recorded', htmlSeeds.length, nonHtmlSeeds.length);
+                // Audit row is written *inside* the `.bak` window: a libsql
+                // hiccup or transient lock on the INSERT aborts the ingestion
+                // and the `.bak` restore wipes the pre-inserted seeds too,
+                // so "either the whole run took or none of it did" holds at
+                // the ingestion boundary. Past behaviour swallowed the
+                // failure post-scrape; the new boundary makes restore safe
+                // and useful, so the swallow is gone (see
+                // {@link CrawlerOrchestrator.#writeInventoryRunRow}).
+                await CrawlerOrchestrator.#writeInventoryRunRow(archive, {
+                    inventoryUrlsCount: inventoryUrls.length,
+                    htmlSeedsCount: htmlSeeds.length,
+                    nonHtmlCount: nonHtmlSeeds.length,
+                    outOfScope,
+                    sourceFileSha256,
+                });
+                // Ingestion's DB writes are now committed. From here on a
+                // throw must NOT trigger the `.bak` restore (it would wipe
+                // the durable seeds + audit row). Setting the flag *before*
+                // the `.bak` unlink covers the rare Windows / antivirus
+                // path where `unlinkFile` itself fails with EBUSY/EPERM —
+                // the `.bak` may leak on disk for the operator to delete
+                // manually, but the archive state stays intact.
+                ingestionComplete = true;
+                // Release `.bak` — ingestion succeeded. Beyond this point a
+                // throw is the scrape phase's problem; the archive stays
+                // intact and the operator runs `--resume` to recover.
+                await ignoreEnoent(unlinkFile(backupPath));
+                // Config sent to the user-facing `initializedCallback`
+                // (matches the rest of the orchestrator's public surface —
+                // no inventory bookkeeping leaks out).
+                const baseConfig = {
+                    ...archived,
+                    ...cleanObject(options),
+                    recursive: true,
+                    fromList: false,
+                };
+                const seedSet = new Set(htmlSeeds.map((u) => u.withoutHashAndAuth));
+                // CrawlConfig overlay handed to the orchestrator constructor —
+                // carries the runtime-only `inventoryMode` that drives source
+                // labelling. Not persisted to the archive.
+                const orchestratorOptions = {
+                    ...baseConfig,
+                    inventoryMode: { seedUrls: seedSet },
+                };
+                if (htmlSeeds.length > 0) {
+                    const orchestrator = new CrawlerOrchestrator(archive, orchestratorOptions);
+                    // Re-read pending *after* the pre-insert so the strict-
+                    // pending set includes the freshly inserted
+                    // `inventory-seed` rows; feed that into `crawler.resume`
+                    // and start a seedless `crawling([])` — the same pattern
+                    // `retryFailed` uses to drive the dealer from the
+                    // pending set alone (see retryFailed's
+                    // `crawling([], { recursive })` invocation).
+                    const { scraped: scrapedAfter, pending: pendingAfter } = await archive.getCrawlingState();
+                    const resources = await archive.getResourceUrlList();
+                    // Pre-existing rendered HTML page count seeds the
+                    // session-spanning `pagesScraped` counter so the progress
+                    // header reads `internalDone(cumulative pagesScraped)`
+                    // rather than session-only — matches the `append` /
+                    // `retryFailed` / `resume` paths and avoids users reading
+                    // the parenthesised number as "inner pages dropped to N".
+                    const pagesScrapedOffset = await archive.getScrapedHtmlPageCount();
+                    orchestrator.#crawler.resume(pendingAfter, scrapedAfter, resources, pagesScrapedOffset);
+                    if (initializedCallback) {
+                        await initializedCallback(orchestrator, baseConfig);
+                    }
+                    log('Start inventory');
+                    log('Archive %s', absFilePath);
+                    log('HTML seeds %O', htmlSeeds.map((u) => u.href));
+                    await CrawlerOrchestrator.#preloadDnsBurnedHostCache(archive);
+                    await orchestrator.crawling([], { recursive: true });
+                    CrawlerOrchestrator.#finalizeCrawlSession();
+                    await archive.setUrlOrder();
+                    return orchestrator;
+                }
+                // Only non-HTML URLs were imported — nothing left to render,
+                // but still update sort order and finalize.
+                const orchestrator = new CrawlerOrchestrator(archive, orchestratorOptions);
+                if (initializedCallback) {
+                    await initializedCallback(orchestrator, baseConfig);
+                }
+                await archive.setUrlOrder();
+                return orchestrator;
+            }
+            catch (error) {
+                if (ingestionComplete) {
+                    // Scrape phase failed; the pre-inserted seeds + audit
+                    // row are durable inside `tmpDir/db.sqlite` but not yet
+                    // on disk as a `.nitpicker` tar. The outer catch below
+                    // runs `archive.close()`, which sees the original
+                    // (pre-inventory) `.nitpicker` already on disk and
+                    // would just `remove(tmpDir)` — silently wiping every
+                    // `inventory-seed` row and the audit row.
+                    //
+                    // Persist the ingested state ourselves before letting
+                    // the outer catch unwind, then re-throw so the operator
+                    // learns about the scrape failure (and can recover via
+                    // `crawl --resume <archive>`). `releaseHandle` shares
+                    // the orchestrator's `#closeOnce` guard, so the outer
+                    // catch's `close()` becomes a no-op for the destructive
+                    // step and only runs `releaseLock` cleanup.
+                    try {
+                        await archive.write();
+                        await archive.releaseHandle();
+                    }
+                    catch (persistError) {
+                        throw new AggregateError([error, persistError], 'inventory scrape phase failed AND persisting the ingested state to disk also failed. The archive may be in an inconsistent state — check tmpDir.');
+                    }
+                    throw error;
+                }
+                try {
+                    await copyFile(backupPath, absFilePath);
+                    await ignoreEnoent(unlinkFile(backupPath));
+                }
+                catch (restoreError) {
+                    throw new AggregateError([error, restoreError], `inventory failed AND restore from backup failed. Original archive backup is left at: ${backupPath}`);
+                }
+                throw error;
+            }
+        }
+        catch (error) {
+            await archive.close().catch(() => { });
+            throw error;
+        }
+    }
+    /**
+     * Re-fetch previously-failed pages in an existing `.nitpicker` archive.
+     *
+     * Opens the archive, resets every page whose previous attempt ended in a
+     * recoverable failure (missing status / content type, or a 5xx status — see
+     * {@link Archive.resetFailedPages}) back to pending, and resumes crawling.
+     * The archived crawl configuration is reused — scopes, excludes, keywords,
+     * user agent, etc. — so the retry honours the original crawl boundaries
+     * unless a field is explicitly overridden via `options`. The exception is
+     * `recursive`: it is taken from `options` (the CLI flag defaults it to
+     * `true`) rather than inherited from the archive, so a retry decides afresh
+     * whether to follow newly-discovered URLs regardless of how the original
+     * crawl was run.
+     *
+     * When `recursive` is enabled (the default), newly-discovered URLs from the
+     * re-fetched pages are followed and crawled from scratch; when disabled, only
+     * the failed pages themselves are re-fetched. The archived roots seed the
+     * crawl scope while the reset pages are picked up through the resumed pending
+     * set, so failed external pages stay external (metadata-only) instead of being
+     * promoted into scope, and a failed root is re-fetched in place.
+     *
+     * A `<archive>.bak` is created before any DB mutation and removed on success;
+     * if the crawl throws, the backup is restored to keep the original archive
+     * intact.
+     *
+     * List-mode archives (`info.fromList === true`) are rejected for the same
+     * reason as {@link CrawlerOrchestrator.append}: their pages are metadata-only.
+     * @param archivePath - Absolute or relative path to the existing `.nitpicker`.
+     * @param options - Optional config overrides applied on top of the archived config.
+     * @param initializedCallback - Optional callback invoked after initialization but before crawling resumes.
+     * @returns The orchestrator instance after the retry crawl completes.
+     * @throws {Error} When the archive is in list mode or has no parseable roots.
+     */
+    static async retryFailed(archivePath, options, initializedCallback) {
+        const cwd = options?.cwd ?? process.cwd();
+        const absFilePath = path.isAbsolute(archivePath)
+            ? archivePath
+            : path.resolve(cwd, archivePath);
+        const archive = await Archive.open({ filePath: absFilePath, cwd });
+        // Any throw between here and the successful return must release the
+        // archive lock and clean up tmpDir; the caller's `close()` only runs on
+        // the happy path.
+        try {
+            const archived = await archive.getConfig();
+            if (archived.fromList) {
+                throw new Error('Cannot retry a list-mode archive: this archive was created with --list/--list-file and contains metadata-only pages. Create a fresh archive instead.');
+            }
+            const rootsParsed = sortUrl(archived.roots, archived);
+            if (rootsParsed.length === 0) {
+                throw new Error('retry: archive has no parseable root URLs');
+            }
+            const config = {
+                ...archived,
+                ...cleanObject(options),
+                roots: archived.roots,
+                fromList: false,
+                baseUrl: archived.baseUrl,
+            };
+            const backupPath = absFilePath + '.bak';
+            await copyFile(absFilePath, backupPath);
+            try {
+                const resetUrls = await archive.resetFailedPages();
+                log('Start retrying failed pages');
+                log('Archive %s', absFilePath);
+                log('Reset %d failed page(s)', resetUrls.length);
+                const orchestrator = new CrawlerOrchestrator(archive, config);
+                const { scraped, pending } = await archive.getCrawlingState();
+                const resources = await archive.getResourceUrlList();
+                const pagesScrapedOffset = await archive.getScrapedHtmlPageCount();
+                orchestrator.#crawler.resume(pending, scraped, resources, pagesScrapedOffset);
+                if (initializedCallback) {
+                    await initializedCallback(orchestrator, config);
+                }
+                await CrawlerOrchestrator.#preloadDnsBurnedHostCache(archive);
+                await orchestrator.crawling([], { recursive: config.recursive });
+                CrawlerOrchestrator.#finalizeCrawlSession();
+                await archive.setUrlOrder();
+                await ignoreEnoent(unlinkFile(backupPath));
+                return orchestrator;
+            }
+            catch (error) {
+                try {
+                    await copyFile(backupPath, absFilePath);
+                    await ignoreEnoent(unlinkFile(backupPath));
+                }
+                catch (restoreError) {
+                    // Restore itself failed — surface both so the operator knows
+                    // the .bak still exists and the original archive may be
+                    // corrupt. The outer `catch` still releases the lock.
+                    throw new AggregateError([error, restoreError], `retry failed AND restore from backup failed. Original archive backup is left at: ${backupPath}`);
+                }
+                throw error;
+            }
+        }
+        catch (error) {
+            await archive.close().catch(() => { });
+            throw error;
+        }
+    }
     /**
      * Resume a previously interrupted crawl from an existing archive file.
      *
@@ -408,7 +921,8 @@ export class CrawlerOrchestrator extends EventEmitter {
         }
         const { scraped, pending } = await archive.getCrawlingState();
         const resources = await archive.getResourceUrlList();
-        orchestrator.#crawler.resume(pending, scraped, resources);
+        const pagesScrapedOffset = await archive.getScrapedHtmlPageCount();
+        orchestrator.#crawler.resume(pending, scraped, resources, pagesScrapedOffset);
         if (initializedCallback) {
             await initializedCallback(orchestrator, config);
         }
@@ -416,9 +930,95 @@ export class CrawlerOrchestrator extends EventEmitter {
         log('Data %s', stubPath);
         log('URL %s', url.href);
         log('Config %O', config);
+        await CrawlerOrchestrator.#preloadDnsBurnedHostCache(archive);
         await orchestrator.crawling([url]);
+        CrawlerOrchestrator.#finalizeCrawlSession();
         return orchestrator;
     }
+    /**
+     * Seeds {@link dnsBurnedHostCache} from `crawl_errors` history at re-open
+     * (append / inventory / retryFailed / resume). Called after Archive.open
+     * succeeds and before crawling starts, so the first URL on a burned host
+     * already short-circuits — no retry budget is spent on a dead host that
+     * the previous crawl already proved was dead.
+     *
+     * Fresh `crawling()` skips this — there is no archive history to seed
+     * from. Within-session learning still kicks in via the `onGiveUp` mark.
+     * @param archive - The opened archive whose `crawl_errors` is read.
+     */
+    static async #preloadDnsBurnedHostCache(archive) {
+        const hosts = await archive.listDnsBurnedHostCandidates();
+        for (const host of hosts) {
+            dnsBurnedHostCache.set(host, 'dns');
+        }
+        if (hosts.length > 0) {
+            // eslint-disable-next-line no-console
+            console.error(`[preload] DNS-burned hosts: ${hosts.length} (will short-circuit subsequent URLs)`);
+        }
+    }
+    /**
+     * Persist one `inventory_runs` audit row inside the ingestion phase of a
+     * `--inventory` invocation, before the `.bak` is released. Lives as a
+     * static helper because the audit-row shape (timestamp stamping + label
+     * auto-gen + the privacy-driven path elision documented below) is a
+     * cohesive concern that benefits from staying outside the long
+     * `inventory()` body even though only one caller remains after the
+     * ingestion-phase consolidation.
+     *
+     * `ran_at` is stamped now (ingestion-completion timestamp; the scrape
+     * phase that may follow is treated as separate). `list_label` is
+     * auto-generated from `ran_at` when the CLI did not pass one — Phase 1
+     * has no `--label` flag, so this is always the auto form.
+     * `source_file_sha256` arrives pre-computed via
+     * `aggregates.sourceFileSha256` (the CLI's `inventoryCrawl` ran
+     * `computeFileSha256` against the input txt before the orchestrator
+     * was even invoked). The orchestrator boundary deliberately never sees
+     * the absolute path — see {@link InventoryRunAggregates} for the
+     * privacy rationale.
+     *
+     * **Audit-write failures abort the ingestion phase.** The earlier
+     * implementation swallowed them because the audit was the last write
+     * after* the scrape, so re-throwing would have wiped a completed crawl;
+     * with audit now lifted into the `.bak`-protected ingestion phase the
+     * trade-off flips. A failed audit row is restorable: the outer catch
+     * copies `.bak` back over the archive and the operator reruns the
+     * (short) ingestion from scratch. That keeps `inventory_runs` honest
+     * (no "ran but unrecorded" rows) at the cost of one rerun.
+     *
+     * Forward-compat: when Phase 2 introduces an explicit `--label` flag,
+     * thread `labelOverride` through {@link inventory} into the `aggregates`
+     * shape so the auto-name can be overridden.
+     * @param archive - The opened archive to write the audit row into.
+     * @param aggregates - The counts captured during the inventory pass; see {@link InventoryRunAggregates}.
+     */
+    static async #writeInventoryRunRow(archive, aggregates) {
+        const ranAt = new Date().toISOString();
+        await archive.recordInventoryRun({
+            ran_at: ranAt,
+            list_label: `inventory-${ranAt}`,
+            source_file_sha256: aggregates.sourceFileSha256,
+            total_lines: aggregates.inventoryUrlsCount,
+            new_pages: aggregates.htmlSeedsCount,
+            new_resources: aggregates.nonHtmlCount,
+            scope_skipped: aggregates.outOfScope,
+        });
+    }
+    /**
+     * Tears down session-scoped crawler caches and prints a short-circuit
+     * summary if any URL fetches were skipped. Invoked at the four
+     * crawl-session boundaries (`crawling` / `append` / `inventory` /
+     * `retryFailed` / `resume`) where the previous `clearDestinationCache`
+     * call already lived.
+     */
+    static #finalizeCrawlSession() {
+        const skipped = dnsBurnedHostShortCircuitCounter.count;
+        if (skipped > 0) {
+            // eslint-disable-next-line no-console
+            console.error(`[preload] Short-circuited ${skipped} URL(s) on DNS-burned hosts`);
+        }
+        clearDestinationCache();
+        clearDnsBurnedHostCache();
+    }
 }
 /**
  * Await a filesystem promise but silently swallow only `ENOENT` errors. Any