@nitpicker/crawler 0.20.0 → 0.22.0

package/lib/crawler-orchestrator.js

@@ -8,9 +8,11 @@ import { APPEND_SETUP_PHASES } from './append-setup-phases.js';
 import Archive from './archive/archive.js';
 import { copyFileWithProgress } from './archive/filesystem/copy-file-with-progress.js';
 import { REQUIRED_FORMAT_VERSION } from './archive/meta/assert-compatible-version.js';
+import { computeAutoRetryBackoffDelayMs } from './compute-auto-retry-backoff-delay.js';
 import { clearDestinationCache } from './crawler/clear-destination-cache.js';
 import { clearDnsBurnedHostCache } from './crawler/clear-dns-burned-host-cache.js';
 import Crawler from './crawler/crawler.js';
+import { buildDedupeCapObservation } from './crawler/dedupe/build-dedupe-cap-observation.js';
 import { dnsBurnedHostCache } from './crawler/dns-burned-host-cache.js';
 import { dnsBurnedHostShortCircuitCounter } from './crawler/dns-burned-host-short-circuit-counter.js';
 import { findScopeEntry } from './crawler/find-scope-entry.js';
@@ -20,8 +22,11 @@ import { PreloadShortCircuitError } from './crawler/preload-short-circuit-error.
 import { protocolAgnosticKey } from './crawler/protocol-agnostic-key.js';
 import { shouldSkipUrl } from './crawler/should-skip-url.js';
 import { crawlerLog, log } from './debug.js';
+import { delayOrAbort } from './delay-or-abort.js';
 import { INVENTORY_SETUP_PHASES } from './inventory-setup-phases.js';
 import { normalizeToArray } from './normalize-to-array.js';
+import { PendingUrlsRemainError } from './pending-urls-remain-error.js';
+import { RECRAWL_SETUP_PHASES } from './recrawl-setup-phases.js';
 import { resolveOutputPath } from './resolve-output-path.js';
 import { resourceRowToLookupResult } from './resource-row-to-lookup-result.js';
 import { RESUME_SETUP_PHASES } from './resume-setup-phases.js';
@@ -29,7 +34,7 @@ import { RETRY_FAILED_SETUP_PHASES } from './retry-failed-setup-phases.js';
 import { SETUP_RECOVERY_PHASE_LABELS } from './setup-recovery-phase-labels.js';
 import { cleanObject } from './utils/object/clean-object.js';
 import { WriteQueue } from './write-queue.js';
-const [RECOVERY_RESTORE_FROM_BACKUP, RECOVERY_PERSIST_INGESTED_STATE] = SETUP_RECOVERY_PHASE_LABELS;
+const [RECOVERY_RESTORE_FROM_BACKUP, RECOVERY_LEAVE_STATE_FOR_RESUME] = SETUP_RECOVERY_PHASE_LABELS;
 /**
  * Default list of external URL prefixes excluded from crawling.
  * Includes social media sharing endpoints that are commonly linked
@@ -73,8 +78,27 @@ export const DEFAULT_EXCLUDED_EXTERNAL_URLS = [
 export class CrawlerOrchestrator extends EventEmitter {
     /** The archive instance for persisting crawl results to SQLite + tar. */
     #archive;
+    /**
+     * Set when the archive's own `'error'` event fires (a DB/storage-level
+     * failure — see the constructor's listener), so
+     * `#crawlUntilPendingClears` can tell "the session ended because pages
+     * are still pending" apart from "the underlying storage broke" (issue
+     * #350). Retrying scrape work cannot fix the latter, so the auto-retry
+     * loop re-throws it immediately instead of burning retry attempts
+     * against it — `crawling()`'s own promise resolves normally either way
+     * (the constructor's listener only aborts the crawler; it does not
+     * reject anything), so this flag is the only way to distinguish the
+     * two after the fact.
+     */
+    #archiveFailure = null;
     /** The crawler engine that discovers and scrapes pages. */
     #crawler;
+    /**
+     * Monotonic counter bumped on every `crawling()` call — see that
+     * method's JSDoc for why its listeners key off this instead of relying
+     * on listener removal.
+     */
+    #crawlGeneration = 0;
     /**
      * `dedupe_cap_events.id` for each shape confirmed capped this session, so
      * `crawlEnd` can look up the right row to finalize with
@@ -86,6 +110,18 @@ export class CrawlerOrchestrator extends EventEmitter {
     #dedupeCapEventIds = new Map();
     /** Whether the crawl was started from a pre-defined URL list (non-recursive mode). */
     #fromList;
+    /**
+     * The caller-owned `Lanes` instance, if any — see `CrawlConfig.lanes`'s
+     * JSDoc. Used by `#crawlUntilPendingClears`'s auto-retry wait to render
+     * through `Lanes#header` instead of `console.error` when present (and
+     * {@link #verbose} is `false` — see that field's JSDoc for why verbose
+     * excludes this path), since an injected `Lanes` stays alive across
+     * auto-retry rounds (unlike `deal()`'s own `Lanes`, which closes each
+     * round) and a bare `console.error` would corrupt its live frame.
+     */
+    #lanes;
+    /** See `CrawlConfig.maxAutoRetry`'s JSDoc. */
+    #maxAutoRetry;
     /**
      * The `network_outages` row id for the currently-open outage, or `null`
      * when none is open. Set by the `networkOutageConfirmed` handler (once
@@ -97,6 +133,41 @@ export class CrawlerOrchestrator extends EventEmitter {
     #openNetworkOutageId = null;
     /** `startedAt` of the currently-open outage, tracked alongside {@link #openNetworkOutageId} so `networkOutageRecovered` can compute a duration for {@link networkOutageSummaryCounter}. */
     #openNetworkOutageStartedAt = null;
+    /**
+     * Set for the duration of {@link updateRuntimeOptions}'s enqueued
+     * `archive.updateConfig()` call — the constructor's `Archive` `'error'`
+     * listener checks this to skip its normal fatal handling (setting
+     * {@link #archiveFailure}, aborting the crawler, emitting `'error'`) for
+     * that one write. A failure there means only that the new
+     * `parallels`/`interval`/exclude values won't survive a later
+     * `--resume`/`--append`/`--retry-failed` — the in-memory change already
+     * took effect (`Crawler#updateRuntimeOptions` runs synchronously, before
+     * this write is even enqueued) and the crawl itself is otherwise
+     * unaffected, so treating it as crawl-fatal (like a real page/resource
+     * write failure, which DOES indicate DB corruption) would let a
+     * transient DB conflict on an optional persistence write cut short a
+     * multi-hour crawl over nothing worse than "an exclude pattern won't be
+     * remembered on resume." Safe without additional locking: every write
+     * this class issues — this one included — is serialized through the same
+     * {@link #writeQueue}, so at most one write (and therefore at most one
+     * meaning for this flag) is ever in flight at a time.
+     */
+    #persistingRuntimeOptionsPatch = false;
+    /**
+     * Mirrors `CrawlerOptions.verbose` (forwarded to `Crawler` at
+     * construction). `#crawlUntilPendingClears`'s auto-retry wait reads this
+     * alongside {@link #lanes}: `Lanes#header()` only queues its text in
+     * verbose mode (prefixed onto the *next* `update()` call) rather than
+     * writing immediately, so routing the auto-retry wait message through it
+     * while verbose would leave that message effectively invisible for the
+     * whole backoff wait — a regression from the unconditional
+     * `console.error` this replaced, which always printed immediately. Verbose
+     * mode falls back to `console.error` instead, matching its pre-`#lanes`
+     * behavior exactly (verbose output is append-only lines anyway, so an
+     * interleaved `console.error` line is harmless there, unlike in
+     * non-verbose mode where it would corrupt the live frame).
+     */
+    #verbose;
     /** Serializes archive writes from crawler event handlers (FIFO). */
     #writeQueue = new WriteQueue();
     /**
@@ -109,15 +180,26 @@ export class CrawlerOrchestrator extends EventEmitter {
     constructor(archive, options) {
         super();
         this.#fromList = !!options?.list;
+        this.#maxAutoRetry = options?.maxAutoRetry ?? 3;
         this.#archive = archive;
         this.#archive.on('error', (e) => {
+            const error = e instanceof Error ? e : new Error(String(e));
+            if (this.#persistingRuntimeOptionsPatch) {
+                // See `#persistingRuntimeOptionsPatch`'s JSDoc for why this one
+                // write's failure does not escalate to `#archiveFailure`/abort/
+                // `'error'` — only debug-logged (`DEBUG=Nitpicker:*`), matching
+                // this class's other best-effort-write logging.
+                log('updateRuntimeOptions: archive.updateConfig failed (non-fatal): %O', error);
+                return;
+            }
+            this.#archiveFailure = error;
             this.#crawler.abort();
             void this.emit('error', {
                 pid: process.pid,
                 isMainProcess: true,
                 url: null,
                 isExternal: false,
-                error: e instanceof Error ? e : new Error(String(e)),
+                error: this.#archiveFailure,
             });
         });
         const defaultUserAgent = `Nitpicker/${pkg.version}`;
@@ -180,12 +262,18 @@ export class CrawlerOrchestrator extends EventEmitter {
             networkProbe: options?.networkProbe ?? null,
             dedupeCap: options?.dedupeCap ?? null,
             dedupeMapCap: options?.dedupeMapCap,
-            // Only the four resuming-session static methods
-            // (`append`/`inventory`/`retryFailed`/`resume`) pass this — a
-            // fresh `crawling()` has no archive history to seed from (see
-            // `CrawlConfig.preloadedStickyShapeKeys`'s JSDoc).
+            // Only the five resuming-session static methods
+            // (`append`/`inventory`/`recrawl`/`retryFailed`/`resume`) pass
+            // this — a fresh `crawling()` has no archive history to seed from
+            // (see `CrawlConfig.preloadedStickyShapeKeys`'s JSDoc).
             preloadedStickyShapeKeys: options?.preloadedStickyShapeKeys ?? [],
+            // Same five methods, same rationale (see
+            // `CrawlConfig.preloadedDedupeObservations`'s JSDoc).
+            preloadedDedupeObservations: options?.preloadedDedupeObservations ?? [],
+            lanes: options?.lanes,
         });
+        this.#lanes = options?.lanes;
+        this.#verbose = options?.verbose ?? false;
     }
     /**
      * Enables `await using orchestrator = ...`. Closes the archive (write
@@ -233,23 +321,74 @@ export class CrawlerOrchestrator extends EventEmitter {
      * Sets up event listeners on the crawler, starts crawling, and resolves
      * when the crawl completes. Discovered pages, external pages, skipped pages,
      * and resources are forwarded to the archive for storage.
+     *
+     * Safe to call more than once on the same instance —
+     * `#crawlUntilPendingClears` (issue #350) re-invokes this for each
+     * auto-retry attempt against the same long-lived `#crawler`. `Crawler`
+     * (`TypedAwaitEventEmitter`) has no listener-removal API, so a second
+     * call cannot replace the first call's listeners — it can only stack
+     * another set alongside them. Every listener this method attaches is
+     * instead guarded by a monotonic generation counter (`isCurrent()`,
+     * defined below): once a later call bumps it, every earlier call's
+     * listeners permanently fail the check and become inert no-ops, leaving
+     * exactly the latest call's listeners actually writing anything.
      * @param list - The list of parsed URLs to crawl. May be empty when a resumed
      *   session already has pending pages queued (for example `--retry-failed`).
      * @param opts - Optional crawl overrides.
      * @param opts.recursive - Whether discovered URLs are followed. Defaults to
      *   `!fromList` (recursive unless the archive was created from a URL list), so
      *   existing callers keep their behaviour; the retry flow passes it explicitly.
+     * @param opts.suppressFlushNotice - Skip emitting `flushingPendingWrites`
+     *   (issue #350). Set by `#crawlUntilPendingClears` for every auto-retry
+     *   attempt after the first: that event starts the CLI's crawl-tail
+     *   `TaskList` (`attach-crawl-display.ts`), which must stay closed until
+     *   the whole retry loop is done — a second `deal()`/`Lanes` cycle
+     *   starting while that `TaskList` is still open would corrupt the
+     *   display (see ARCHITECTURE.md's `Lanes`/`Display` single-instance
+     *   invariant). The write-queue drain itself is unaffected; only the
+     *   CLI-facing progress event is skipped.
+     * @param opts.isRetryContinuation - Forwarded to `Crawler#start()`
+     *   (issue #350). Set by `#crawlUntilPendingClears` for every auto-retry
+     *   attempt after the first, so `#runDeal` preserves cross-attempt
+     *   learned state (known-good hosts, outage-detector window) instead of
+     *   discarding it as if this were an unrelated fresh session.
      * @returns A promise that resolves when crawling is complete.
      */
     async crawling(list, opts) {
         const writeQueue = this.#writeQueue;
+        // Same value passed to `Crawler#start()` below (line ~822) — computed
+        // once here so the `setPage`/`setExternalPage` listeners can forward
+        // it for `is_metadata_only` (#369) without recomputing the default.
+        const recursive = opts?.recursive ?? !this.#fromList;
         // Per-session state, like `Crawler`'s own `#successfulHosts.clear()` /
         // `#networkGate.open()` reset at the start of `#runDeal` — a fresh
         // session must not inherit a dangling outage id from a prior one.
         this.#openNetworkOutageId = null;
         this.#openNetworkOutageStartedAt = null;
+        // `Crawler` (`TypedAwaitEventEmitter`) has no listener-removal API, so
+        // a second call on the same instance (an auto-retry attempt, issue
+        // #350) cannot replace the first call's listeners — it can only add
+        // another set alongside them. Every listener below is instead guarded
+        // by `isCurrent()`, keyed off a monotonic generation counter bumped
+        // here: once a later call starts, every earlier call's listeners
+        // permanently fail this check and become inert no-ops (their
+        // `writeQueue`/`#archive` side effects never run), leaving exactly
+        // one "live" set — this call's — actually writing anything.
+        const generation = ++this.#crawlGeneration;
+        const isCurrent = () => generation === this.#crawlGeneration;
+        // The ONLY way any listener below reaches `this.#crawler.on()` — a
+        // structural guarantee (not a per-listener discipline a future edit
+        // could forget) that no handler can run without the `isCurrent()`
+        // check, however many are added here in the future.
+        const registerGuarded = (event, handler) => {
+            this.#crawler.on(event, (payload) => {
+                if (!isCurrent())
+                    return;
+                handler(payload);
+            });
+        };
         return new Promise((resolve, reject) => {
-            this.#crawler.on('error', (error) => {
+            registerGuarded('error', (error) => {
                 if (error.error instanceof PreloadShortCircuitError) {
                     // DNS-burned host short-circuit: the underlying cause already
                     // lives in `crawl_errors` from the original DNS failure.
@@ -268,33 +407,33 @@ export class CrawlerOrchestrator extends EventEmitter {
                     .catch((writeError) => reject(writeError));
                 void this.emit('error', error);
             });
-            this.#crawler.on('page', ({ result, source, bodyHash }) => {
+            registerGuarded('page', ({ result, source, bodyHash }) => {
                 writeQueue
-                    .enqueue(() => this.#archive.setPage(result, source, bodyHash))
+                    .enqueue(() => this.#archive.setPage(result, source, bodyHash, recursive))
                     .catch((error) => reject(error));
             });
-            this.#crawler.on('externalPage', ({ result, source }) => {
+            registerGuarded('externalPage', ({ result, source }) => {
                 writeQueue
-                    .enqueue(() => this.#archive.setExternalPage(result, source))
+                    .enqueue(() => this.#archive.setExternalPage(result, source, recursive))
                     .catch((error) => reject(error));
             });
-            this.#crawler.on('skip', ({ url, reason, isExternal }) => {
+            registerGuarded('skip', ({ url, reason, isExternal }) => {
                 writeQueue
                     .enqueue(() => this.#archive.setSkippedPage(url, reason, isExternal))
                     .catch((error) => reject(error));
             });
-            this.#crawler.on('pageError', ({ url, phase, message, isExternal }) => {
+            registerGuarded('pageError', ({ url, phase, message, isExternal }) => {
                 writeQueue
                     .enqueue(() => this.#archive.addPageError(url, phase, message, isExternal))
                     .catch((error) => reject(error));
             });
-            this.#crawler.on('redirect', ({ result, source }) => {
+            registerGuarded('redirect', ({ result, source }) => {
                 writeQueue
                     .enqueue(() => this.#archive.setRedirect(result, source))
                     .catch((error) => reject(error));
                 void this.emit('redirect', { result });
             });
-            this.#crawler.on('networkOutageConfirmed', ({ startedAt, detectedAt, probeHost, triggerErrorCount, triggerHostCount }) => {
+            registerGuarded('networkOutageConfirmed', ({ startedAt, detectedAt, probeHost, triggerErrorCount, triggerHostCount }) => {
                 crawlerLog('Network outage confirmed: probeHost=%s triggerErrorCount=%d triggerHostCount=%d', probeHost, triggerErrorCount, triggerHostCount);
                 // Rare anomaly notice, printed unconditionally even though this
                 // fires while `deal()`'s own crawl-time `Lanes` is actively
@@ -324,7 +463,7 @@ export class CrawlerOrchestrator extends EventEmitter {
                 })
                     .catch((error) => reject(error));
             });
-            this.#crawler.on('networkOutageRecovered', ({ endedAt }) => {
+            registerGuarded('networkOutageRecovered', ({ endedAt }) => {
                 // The `id` read is deferred to INSIDE the queued closure, not
                 // read synchronously here, because `networkOutageConfirmed`'s
                 // INSERT is itself only queued (not awaited) when that event
@@ -360,7 +499,7 @@ export class CrawlerOrchestrator extends EventEmitter {
                 })
                     .catch((error) => reject(error));
             });
-            this.#crawler.on('dedupeCap', ({ shapeKey, sampleUrl, bodyHash, effectiveThreshold, observedCount }) => {
+            registerGuarded('dedupeCap', ({ shapeKey, sampleUrl, bodyHash, effectiveThreshold, observedCount }) => {
                 crawlerLog('Dedupe cap reached: shapeKey=%s effectiveThreshold=%d observedCount=%d', shapeKey, effectiveThreshold, observedCount);
                 console.error(`[dedupe-cap] same-cluster trap confirmed: ${shapeKey} (sample: ${sampleUrl})`);
                 writeQueue
@@ -377,27 +516,27 @@ export class CrawlerOrchestrator extends EventEmitter {
                 })
                     .catch((error) => reject(error));
             });
-            this.#crawler.on('response', ({ resource, source }) => {
+            registerGuarded('response', ({ resource, source }) => {
                 writeQueue
                     .enqueue(() => this.#archive.setResources(resource, source))
                     .catch((error) => reject(error));
             });
-            this.#crawler.on('responseReferrers', (resource) => {
+            registerGuarded('responseReferrers', (resource) => {
                 writeQueue
                     .enqueue(() => this.#archive.setResourcesReferrers(resource))
                     .catch((error) => reject(error));
             });
-            this.#crawler.on('consoleLogs', ({ pageUrl, redirectPaths, entries }) => {
+            registerGuarded('consoleLogs', ({ pageUrl, redirectPaths, entries }) => {
                 writeQueue
                     .enqueue(() => this.#archive.setConsoleLogs(pageUrl, redirectPaths, entries))
                     .catch((error) => reject(error));
             });
-            this.#crawler.on('crawlEnd', () => {
+            registerGuarded('crawlEnd', () => {
                 // Read BEFORE enqueuing the dedupeCap-finalize closure below
                 // (issue #294) so this reflects genuine backlog from the
                 // crawl's own page/resource writes, not the finalize
                 // closure's own, always-present entry.
-                if (writeQueue.pending > 0) {
+                if (writeQueue.pending > 0 && !opts?.suppressFlushNotice) {
                     void this.emit('flushingPendingWrites', { pending: writeQueue.pending });
                 }
                 // Deferred to INSIDE a queued closure, not read synchronously
@@ -450,7 +589,10 @@ export class CrawlerOrchestrator extends EventEmitter {
                     .then(() => resolve())
                     .catch((error) => reject(error));
             });
-            this.#crawler.start(list, { recursive: opts?.recursive ?? !this.#fromList });
+            this.#crawler.start(list, {
+                recursive,
+                isRetryContinuation: opts?.isRetryContinuation,
+            });
         });
     }
     /**
@@ -482,6 +624,58 @@ export class CrawlerOrchestrator extends EventEmitter {
     getUndeadPid() {
         return this.#crawler.getUndeadPid();
     }
+    /**
+     * Applies a runtime change to the in-progress crawl's tunable options
+     * (`parallels`/`interval`/the three exclude arrays — see
+     * {@link Crawler.updateRuntimeOptions}) and persists the resulting
+     * values to the archive's `info` row so a later `--resume` / `--append`
+     * / `--retry-failed` reuses them (`Archive#updateConfig`).
+     *
+     * The in-memory change takes effect immediately, before this method
+     * returns. The archive write is enqueued on {@link #writeQueue} — so it
+     * is ordered relative to the crawl's own page/resource writes — but not
+     * awaited: this method is synchronous so a caller (e.g. the CLI reading
+     * a console command) can report the new value back without waiting on
+     * disk I/O. A failure in that write does NOT abort the crawl or set
+     * {@link #archiveFailure} — see {@link #persistingRuntimeOptionsPatch}'s
+     * JSDoc for why this one write is deliberately exempted from the fatal
+     * handling every other archive write gets; it is only debug-logged
+     * (`DEBUG=Nitpicker:*`).
+     * @param patch - The runtime change to apply.
+     * @returns A snapshot of the tunable options after applying `patch`.
+     * @throws {RangeError} If `parallels` is present and not an integer `>= 1`, or `interval` is present and not an integer `>= 0`.
+     * @throws {TypeError} If any exclude entry is present and not a non-empty string.
+     * @example
+     * ```ts
+     * const snapshot = orchestrator.updateRuntimeOptions({ parallels: 4 });
+     * console.log(snapshot.parallels); // 4
+     * ```
+     */
+    updateRuntimeOptions(patch) {
+        const snapshot = this.#crawler.updateRuntimeOptions(patch);
+        this.#writeQueue
+            .enqueue(async () => {
+            this.#persistingRuntimeOptionsPatch = true;
+            try {
+                await this.#archive.updateConfig({
+                    parallels: snapshot.parallels,
+                    interval: snapshot.interval,
+                    excludes: [...snapshot.excludes],
+                    excludeUrls: [...snapshot.excludeUrls],
+                    excludeKeywords: [...snapshot.excludeKeywords],
+                });
+            }
+            finally {
+                this.#persistingRuntimeOptionsPatch = false;
+            }
+        })
+            // The `Archive` `'error'` listener above already logs a failure
+            // here (non-fatal, see `#persistingRuntimeOptionsPatch`) — this
+            // catch exists only to prevent an unhandled rejection from this
+            // fire-and-forget enqueue.
+            .catch(() => { });
+        return snapshot;
+    }
     /**
      * Write the archive to its configured file path.
      *
@@ -509,6 +703,195 @@ export class CrawlerOrchestrator extends EventEmitter {
         });
         void this.emit('writeFileEnd', { filePath: this.#archive.filePath });
     }
+    /**
+     * Releases the archive handle (lock dropped, tmpDir left intact — see
+     * {@link Archive.releaseHandle}) and throws a {@link PendingUrlsRemainError}
+     * (issue #350) describing why `#crawlUntilPendingClears` gave up. Typed
+     * to return `never` so callers can `return this.#abandonPendingRetryLoop(...)`
+     * and satisfy control-flow analysis without an unreachable trailing
+     * `throw`.
+     * @param params - See the matching {@link PendingUrlsRemainError} field for each property's meaning.
+     * @param params.pendingCount - Pending URL count at the moment of giving up.
+     * @param params.attemptsMade - Auto-retry attempts actually run before giving up.
+     * @param params.reason - See {@link PendingUrlsRemainError}'s `reason` field.
+     */
+    async #abandonPendingRetryLoop(params) {
+        const { pendingCount, attemptsMade, reason } = params;
+        await this.#archive.releaseHandle();
+        throw new PendingUrlsRemainError({
+            pendingCount,
+            attemptsMade,
+            maxAutoRetry: this.#maxAutoRetry,
+            reason,
+            stubPath: this.#archive.tmpDir,
+        });
+    }
+    /**
+     * Runs `crawling()` and, if the session ends with pages still pending
+     * (issue #350), automatically re-queues them and re-runs the crawl loop
+     * up to `#maxAutoRetry` times with an exponential backoff between
+     * attempts ({@link computeAutoRetryBackoffDelayMs}) before giving up.
+     *
+     * Every one of the six session-starting static factories
+     * (`crawling`/`append`/`inventory`/`recrawl`/`retryFailed`/`resume`)
+     * routes its first `crawling()` call through here instead of calling it
+     * directly, so that **a `.nitpicker` file existing on disk always
+     * implies `pending === 0`**: this method never lets a factory reach
+     * `orchestrator.write()` (called later by the CLI's post-crawl step)
+     * while pages remain unscraped. When retrying cannot (or should not)
+     * continue, it releases the archive handle — leaving the stub (tmpDir)
+     * on disk, un-packaged, lock released — and throws
+     * {@link PendingUrlsRemainError} so the operator can recover via
+     * `crawl --resume` or `--retry-failed`.
+     *
+     * Three conditions end the loop early, before `#maxAutoRetry` is reached:
+     * - The archive itself failed (`#archiveFailure` set by the constructor's
+     *   `Archive` `'error'` listener — a DB/storage-level failure). Retrying
+     *   scrape work cannot fix a broken database, so this re-throws the
+     *   original failure immediately without releasing-and-wrapping it in a
+     *   `PendingUrlsRemainError` — there is nothing "pending-remains"-shaped
+     *   about a storage failure.
+     * - The crawl was explicitly aborted (`#crawler.signal.aborted` — the
+     *   public `abort()` method, e.g. a caller-driven cancellation or a
+     *   Ctrl+C proxy in tests). `AbortController.signal` cannot be
+     *   un-aborted, so every subsequent `crawling()` call on this same
+     *   `#crawler` would deal zero work forever — retrying would just waste
+     *   one full backoff wait before "no progress" gives up anyway. This
+     *   returns immediately instead, matching this method's pre-#350
+     *   behaviour for an explicit abort: the caller gets the orchestrator
+     *   back with pending possibly `> 0` and decides for itself (the CLI's
+     *   own SIGINT handler never reaches this far — see `crawl.ts` — so the
+     *   `.nitpicker` ⟹ pending = 0 invariant still holds for that path). The
+     *   backoff wait itself is also abort-interruptible ({@link delayOrAbort}
+     *   rather than a bare `delay()`, issue #350 code review): a library
+     *   consumer calling `abort()` mid-wait (unlike the CLI's SIGINT path)
+     *   must not be stuck waiting up to 5 minutes for nothing.
+     * - An attempt makes no dent in the pending count (unchanged or grown):
+     *   burning the remaining budget against a cause retrying will not fix
+     *   (e.g. a wholesale host outage) just delays the operator finding out.
+     *   Checked AFTER the exhaustion check below it in the loop body so
+     *   that a final attempt which is both exhausted AND made no progress
+     *   reports as `'exhausted'` — the more actionable of the two (it tells
+     *   the operator the budget, not just that this one attempt stalled).
+     *
+     * Each retry attempt re-reads `getCrawlingState()` and
+     * `getResourceUrlList()` in full — the same cost `retryFailed`/`resume`/
+     * `append`/`inventory` already pay once per invocation, now paid up to
+     * `#maxAutoRetry` additional times (bounded, default 3). Retry attempts
+     * pass `isRetryContinuation: true` through to `Crawler#start()` so
+     * `#runDeal` preserves cross-attempt learned state (known-good hosts,
+     * network-outage detector window) instead of discarding it as if this
+     * were an unrelated fresh session (issue #350 code review) — the whole
+     * point of retrying is to avoid re-paying that detection cost.
+     * @param list - Forwarded to the first `crawling()` call.
+     * @param opts - Forwarded to the first `crawling()` call.
+     * @param opts.recursive
+     */
+    async #crawlUntilPendingClears(list, opts) {
+        await this.crawling(list, opts);
+        if (this.#archiveFailure) {
+            throw this.#archiveFailure;
+        }
+        if (this.#crawler.signal.aborted) {
+            return;
+        }
+        // `list` is this call's own root/seed URLs — always a full-scrape
+        // target regardless of `is_metadata_only` (a root added via
+        // `crawler.ts`'s direct seeding path never goes through
+        // `processAnchors`, so the anchor-derived flag on its row, if any,
+        // reflects some OTHER page's anchor to the same URL, not this URL's
+        // own root status). Excluded here rather than never written in the
+        // first place — see `replaceAnchorEdges`'s `recursive` doc for why a
+        // root's `is_metadata_only` can be wrongly set to `1` by a
+        // same-crawl page that happens to link to it before it is scraped
+        // itself (#369).
+        const rootKeys = new Set(list.map((u) => u.withoutHashAndAuth));
+        // Fetched at most once across the whole retry loop (issue #350 code
+        // review), not per attempt: `getResourceUrlList()` is a full scan of
+        // every known resource URL, but `Crawler#resume()`'s use of it is
+        // just seeding the in-memory `#resources` Set — idempotent, and
+        // already kept current independently as the live crawl writes new
+        // resources during each attempt. Re-fetching the full list on every
+        // attempt would re-pay that scan cost for no benefit on a large
+        // archive. `undefined` until the first retry actually needs it, so
+        // the common case (pending clears without ever retrying) never
+        // fetches it at all.
+        let cachedResources;
+        let previousPendingCount = null;
+        for (let attempt = 1;; attempt++) {
+            // `pendingMetadataOnly` defaults to `[]`: `.filter()` runs on it a
+            // few lines below, and a test-mocked `Archive` may return
+            // `{ scraped, pending }` without the field (production's real
+            // `getCrawlingState()` always includes it).
+            const { scraped, pending, pendingMetadataOnly = [], } = await this.#archive.getCrawlingState();
+            if (pending.length === 0) {
+                return;
+            }
+            if (attempt > this.#maxAutoRetry) {
+                return this.#abandonPendingRetryLoop({
+                    pendingCount: pending.length,
+                    attemptsMade: attempt - 1,
+                    reason: 'exhausted',
+                });
+            }
+            if (previousPendingCount !== null && pending.length >= previousPendingCount) {
+                return this.#abandonPendingRetryLoop({
+                    pendingCount: pending.length,
+                    attemptsMade: attempt - 1,
+                    reason: 'no-progress',
+                });
+            }
+            previousPendingCount = pending.length;
+            const delayMs = computeAutoRetryBackoffDelayMs(attempt);
+            void this.emit('autoRetryWaiting', {
+                attempt,
+                maxAttempts: this.#maxAutoRetry,
+                pendingCount: pending.length,
+                delayMs,
+            });
+            // A caller-injected `#lanes` (see `CrawlConfig.lanes`'s JSDoc)
+            // stays alive across auto-retry rounds — unlike `deal()`'s own
+            // Lanes, which closes each round — so a bare `console.error`
+            // here would corrupt its still-live frame. Route through
+            // `header()` instead; its `%countdown(...)%` placeholder
+            // animates via the Lanes' own redraw loop, same as the
+            // per-URL interval wait in `crawler.ts`. `#verbose` excludes
+            // this path (see that field's JSDoc: `header()` only queues its
+            // text for the next `update()` call in verbose mode, so it would
+            // go effectively unseen for the whole wait). Without an injected
+            // `#lanes` (or while verbose), this always fires in the gap
+            // after `deal()`'s own Lanes has closed and before the next
+            // round's opens (the retry's own `crawling()` call hasn't run
+            // yet), so there is no active display to corrupt and
+            // `console.error` is safe.
+            if (this.#lanes && !this.#verbose) {
+                this.#lanes.header(`[auto-retry] ${pending.length} pending page(s) remain — retry ${attempt}/${this.#maxAutoRetry} in %countdown(${delayMs},autoRetry,s)%s`);
+            }
+            else {
+                // eslint-disable-next-line no-console -- see comment above
+                console.error(`[auto-retry] ${pending.length} pending page(s) remain — waiting ${Math.round(delayMs / 1000)}s before retry ${attempt}/${this.#maxAutoRetry}`);
+            }
+            await delayOrAbort(delayMs, this.#crawler.signal);
+            if (this.#crawler.signal.aborted) {
+                return;
+            }
+            cachedResources ??= await this.#archive.getResourceUrlList();
+            const pagesScrapedOffset = await this.#archive.getScrapedHtmlPageCount();
+            const metadataOnlyUrls = pendingMetadataOnly.filter((url) => !rootKeys.has(url));
+            this.#crawler.resume(pending, scraped, cachedResources, pagesScrapedOffset, metadataOnlyUrls);
+            await this.crawling([], {
+                recursive: false,
+                suppressFlushNotice: true,
+                isRetryContinuation: true,
+            });
+            if (this.#archiveFailure) {
+                throw this.#archiveFailure;
+            }
+            if (this.#crawler.signal.aborted) {
+                return;
+            }
+        }
+    }
     /**
      * Assign natural URL sort order to every internal page, relaying chunk
      * progress through the `sortingUrls` event (issue #294). Always runs
@@ -538,6 +921,7 @@ export class CrawlerOrchestrator extends EventEmitter {
      * @param initializedCallback - Optional callback invoked after initialization but before crawling starts.
      * @returns A promise that resolves to the CrawlerOrchestrator instance after crawling completes.
      * @throws {Error} If the URL list is empty or contains no valid URLs.
+     * @throws {PendingUrlsRemainError} When the crawl session ends with pages still pending after exhausting auto-retry.
      */
     static async crawling(url, options, initializedCallback) {
         const list = sortUrl(url, options);
@@ -585,6 +969,7 @@ export class CrawlerOrchestrator extends EventEmitter {
             userAgent: options?.userAgent || defaultUserAgent,
             ignoreRobots: options?.ignoreRobots ?? false,
             mainContentSelector: options?.mainContentSelector ?? null,
+            ...buildCreatedCwdPatch(cwd),
         });
         const orchestrator = new CrawlerOrchestrator(archive, {
             ...options,
@@ -597,7 +982,7 @@ export class CrawlerOrchestrator extends EventEmitter {
         log('Start crawling');
         log('URL %O', list.map((url) => url.href));
         log('Config %O', config);
-        await orchestrator.crawling(list);
+        await orchestrator.#crawlUntilPendingClears(list);
         log('Crawling completed');
         CrawlerOrchestrator.#finalizeCrawlSession(orchestrator);
         log('Set order natural URL sort');
@@ -614,7 +999,11 @@ export class CrawlerOrchestrator extends EventEmitter {
      * the expanded scope are demoted back to "needs scraping" so the next pass
      * re-fetches them as full internal pages. A `<archive>.bak` is created
      * before the crawl and removed on success; if the crawl throws, the backup
-     * is restored to keep the original archive intact.
+     * is restored to keep the original archive intact — except when the crawl
+     * ends with {@link PendingUrlsRemainError} (issue #350), where the
+     * un-packaged stub itself is the recovery path and the backup is instead
+     * left untouched (deleted, not restored — see
+     * {@link CrawlerOrchestrator.#abandonBackupOnPendingRemains}).
      *
      * List-mode archives (`info.fromList === true`) are rejected because their
      * pages are all metadata-only and cannot host a recursive append.
@@ -628,6 +1017,7 @@ export class CrawlerOrchestrator extends EventEmitter {
      *   this can't go through the orchestrator's event emitter (issue #294).
      * @returns The orchestrator instance after the append crawl completes.
      * @throws {Error} When `newUrls` is empty, the archive is in list mode, or it cannot be parsed.
+     * @throws {PendingUrlsRemainError} When the crawl session ends with pages still pending after exhausting auto-retry.
      */
     static async append(archivePath, newUrls, options, initializedCallback, setupProgress) {
         const [PHASE_EXTRACTING, PHASE_LOADING_CONFIG, PHASE_BACKING_UP, PHASE_REPROMOTING, PHASE_LOADING_DEDUPE_KEYS, PHASE_LOADING_CRAWL_STATE, PHASE_LOADING_RESOURCES, PHASE_LOADING_SCRAPED_COUNT, PHASE_RESTORING_CRAWL_STATE,] = APPEND_SETUP_PHASES;
@@ -671,6 +1061,12 @@ export class CrawlerOrchestrator extends EventEmitter {
                 fromList: false,
                 recursive: true,
                 baseUrl: mergedRoots[0],
+                // Stamped for `Archive.resume` (issue #350) — this session's
+                // cwd, not `options.cwd` (already spread above and dropped by
+                // `updateConfig`'s allowlist): a stub left behind by THIS
+                // append should resume back to where THIS command ran, not
+                // wherever the original crawl happened to run from.
+                ...buildCreatedCwdPatch(cwd),
             };
             const backupPath = absFilePath + '.bak';
             setupProgress?.onPhase?.(PHASE_BACKING_UP);
@@ -689,22 +1085,52 @@ export class CrawlerOrchestrator extends EventEmitter {
                 await archive.repromoteExternalPages(scopeMap, archived, setupProgress?.onChunkProgress);
                 // Seed the sticky set from prior sessions' confirmed traps so
                 // `--append` does not pay the cost of re-discovering them (see
-                // `DedupeCapTracker`'s constructor JSDoc).
+                // `DedupeCapTracker`'s constructor JSDoc), and replay every
+                // not-yet-capped shape's prior observations so its counter
+                // does not restart at 0 (see `#preloadDedupeCapObservations`).
                 setupProgress?.onPhase?.(PHASE_LOADING_DEDUPE_KEYS);
-                const preloadedStickyShapeKeys = await archive.listDedupeCapShapeKeys();
+                // Independent reads (no data dependency) — run concurrently
+                // rather than paying two sequential round-trips, the same
+                // reasoning `resetFailedPages` already applies to its own
+                // unrelated reads.
+                const [preloadedStickyShapeKeys, preloadedDedupeObservations] = await Promise.all([
+                    archive.listDedupeCapShapeKeys(),
+                    CrawlerOrchestrator.#preloadDedupeCapObservations(archive, options?.dedupeCap ?? null, setupProgress?.onChunkProgress),
+                ]);
                 const orchestrator = new CrawlerOrchestrator(archive, {
                     ...mergedConfig,
                     roots: mergedRoots,
                     preloadedStickyShapeKeys,
+                    preloadedDedupeObservations,
                 });
                 setupProgress?.onPhase?.(PHASE_LOADING_CRAWL_STATE);
-                const { scraped, pending } = await archive.getCrawlingState();
+                // `mergedConfig.recursive` is forced `true` above, so THIS
+                // session's own anchor discovery can only mark an EXTERNAL
+                // anchor as metadata-only. But `pendingMetadataOnly` can also
+                // carry a STALE flag from a PRIOR session — e.g. a URL that
+                // was external (or discovered under a then-non-recursive
+                // config) before this `--append` call promotes it into
+                // `mergedRoots`. `crawler.resume()` below runs BEFORE the new
+                // roots are added via `#crawlUntilPendingClears(newParsed)`
+                // (→ `LinkList#add()`), so `LinkList#add()`'s dedup would
+                // silently let the stale metadataOnly flag win over the
+                // root's full-scrape intent without this exclusion (issue
+                // #369 code review — same class of bug the fix itself
+                // targets). Re-parse `mergedRoots` (`ExURL#withoutHash` form)
+                // to `withoutHashAndAuth` before comparing, matching how
+                // `pendingMetadataOnly` entries are normalised (see the
+                // list-mode `resume`'s identical comment).
+                const { scraped, pending, pendingMetadataOnly = [], } = await archive.getCrawlingState();
+                const mergedRootKeys = new Set(mergedRoots
+                    .map((root) => parseUrl(root, mergedConfig)?.withoutHashAndAuth)
+                    .filter((root) => root !== undefined));
+                const metadataOnlyUrls = pendingMetadataOnly.filter((url) => !mergedRootKeys.has(url));
                 setupProgress?.onPhase?.(PHASE_LOADING_RESOURCES);
                 const resources = await archive.getResourceUrlList(setupProgress?.onChunkProgress);
                 setupProgress?.onPhase?.(PHASE_LOADING_SCRAPED_COUNT);
                 const pagesScrapedOffset = await archive.getScrapedHtmlPageCount();
                 setupProgress?.onPhase?.(PHASE_RESTORING_CRAWL_STATE);
-                orchestrator.#crawler.resume(pending, scraped, resources, pagesScrapedOffset);
+                orchestrator.#crawler.resume(pending, scraped, resources, pagesScrapedOffset, metadataOnlyUrls);
                 if (initializedCallback) {
                     await initializedCallback(orchestrator, mergedConfig);
                 }
@@ -713,13 +1139,17 @@ export class CrawlerOrchestrator extends EventEmitter {
                 log('New roots %O', newRoots);
                 log('Merged roots %O', mergedRoots);
                 await CrawlerOrchestrator.#preloadDnsBurnedHostCache(archive);
-                await orchestrator.crawling(newParsed);
+                await orchestrator.#crawlUntilPendingClears(newParsed);
                 CrawlerOrchestrator.#finalizeCrawlSession(orchestrator);
                 await orchestrator.#setUrlOrder();
                 await ignoreEnoent(unlinkFile(backupPath));
                 return orchestrator;
             }
             catch (error) {
+                if (error instanceof PendingUrlsRemainError) {
+                    await CrawlerOrchestrator.#abandonBackupOnPendingRemains(setupProgress, backupPath);
+                    throw error;
+                }
                 try {
                     setupProgress?.onPhase?.(RECOVERY_RESTORE_FROM_BACKUP);
                     await copyFileWithProgress(backupPath, absFilePath, setupProgress?.onCopyProgress);
@@ -823,6 +1253,7 @@ export class CrawlerOrchestrator extends EventEmitter {
      *   #294).
      * @returns The orchestrator instance after a successful inventory pass.
      * @throws {Error} When `inventoryUrls` is empty or the archive is in list mode. Unresolved pending URLs from a previous crawl do NOT throw — see step 3.
+     * @throws {PendingUrlsRemainError} When the crawl session ends with pages still pending after exhausting auto-retry.
      */
     static async inventory(archivePath, inventoryUrls, options, initializedCallback, source = null, setupProgress) {
         const [PHASE_EXTRACTING, PHASE_LOADING_CONFIG, PHASE_LOADING_CRAWL_STATE_PRE, PHASE_CHECKING_KNOWN_URLS, PHASE_BACKING_UP, PHASE_RECORDING_NON_HTML, PHASE_RECORDING_HTML_SEEDS, PHASE_RECORDING_EXCLUDED, PHASE_LOADING_CRAWL_STATE_POST, PHASE_LOADING_RESOURCES, PHASE_LOADING_SCRAPED_COUNT, PHASE_RESTORING_CRAWL_STATE,] = INVENTORY_SETUP_PHASES;
@@ -849,6 +1280,10 @@ export class CrawlerOrchestrator extends EventEmitter {
             if (archived.fromList) {
                 throw new Error('Cannot run inventory on a list-mode archive: this archive was created with --list/--list-file and contains metadata-only pages. Create a fresh archive instead.');
             }
+            // Stamped for `Archive.resume` (issue #350) — a stub left behind
+            // by THIS inventory run should resume back to where THIS command
+            // ran, independent of `--resume`'s own invocation directory.
+            await archive.updateConfig(buildCreatedCwdPatch(cwd));
             setupProgress?.onPhase?.(PHASE_LOADING_CRAWL_STATE_PRE);
             const { pending } = await archive.getCrawlingState();
             if (pending.length > 0) {
@@ -894,84 +1329,16 @@ export class CrawlerOrchestrator extends EventEmitter {
             if (source) {
                 await archive.saveInventorySourceList(source.sha256, source.bytes);
             }
-            // Parse + scope-classify the candidate URLs. sortUrl drops
-            // unparseable strings; findScopeEntry separates in-scope from
-            // out-of-scope.
-            const parsedAll = sortUrl(inventoryUrls, archived);
-            const scopeMap = new Map();
-            for (const raw of archived.roots) {
-                const parsed = parseUrl(raw, archived);
-                if (!parsed)
-                    continue;
-                const existing = scopeMap.get(parsed.hostname) ?? [];
-                scopeMap.set(parsed.hostname, [...existing, parsed]);
-            }
-            const inScope = [];
-            let outOfScope = 0;
-            for (const url of parsedAll) {
-                if (findScopeEntry(url, scopeMap, archived) === null) {
-                    outOfScope++;
-                }
-                else {
-                    inScope.push(url);
-                }
-            }
-            if (outOfScope > 0) {
-                log('[inventory] %d URL(s) skipped (outside archived scope: %O)', outOfScope, archived.roots);
-            }
-            // Drop URLs that are already represented in the archive (either
-            // as pages or resources). Comparison key is `withoutHashAndAuth`
-            // to mirror what `resolveContentItemId` / `insertResource` actually store.
-            // Two independent reads — Promise.all halves the wait on large
-            // archives where each `WHERE url IN (?)` chunk costs real I/O.
+            // Parse, scope-classify, and split the candidate URLs into
+            // already-known vs. novel — shared with `recrawl`, see
+            // `#classifyInventoryCandidateUrls`.
             setupProgress?.onPhase?.(PHASE_CHECKING_KNOWN_URLS);
-            const candidateUrls = inScope.map((u) => u.withoutHashAndAuth);
-            const [existingPageUrlList, existingResourceUrlList] = await Promise.all([
-                archive.getExistingPageUrls(candidateUrls),
-                archive.getExistingResourceUrls(candidateUrls),
-            ]);
-            const existingPageUrls = new Set(existingPageUrlList);
-            const existingResourceUrls = new Set(existingResourceUrlList);
-            const novelUrls = inScope.filter((u) => {
-                const key = u.withoutHashAndAuth;
-                return !existingPageUrls.has(key) && !existingResourceUrls.has(key);
-            });
-            const knownCount = existingPageUrls.size + existingResourceUrls.size;
-            log('[inventory] %d in-scope, %d already in archive, %d new', inScope.length, knownCount, novelUrls.length);
-            // Split the novel URLs on the exclusion config BEFORE the
-            // HTML/non-HTML classification, so an exclude-matched URL is
-            // recorded as a terminal skipped page instead of being imported
-            // (issue #260). The inputs mirror the scrape phase's fetch-time
-            // gate (`shouldSkipUrl` in `crawler.ts` fed by the constructor's
-            // merge): archived config overlaid with this run's overrides,
-            // and `DEFAULT_EXCLUDED_EXTERNAL_URLS` merged ahead of the
-            // user's prefixes — classification and gate must never disagree
-            // about the same URL. Running this AFTER the known-URL filter is
-            // deliberate: a previously crawled row that newly matches the
-            // exclusion config stays untouched (crawled-wins), matching how
-            // `getExistingPageUrls` shields known rows from re-labelling.
-            // `excludeKeywords` is deliberately absent: it matches rendered
-            // page content, which a URL list does not have — HTML seeds
-            // still get it at render time via the browser verdict.
+            const { outOfScope, novelUrls } = await CrawlerOrchestrator.#classifyInventoryCandidateUrls(inventoryUrls, archived, archive);
+            // Split the novel URLs on the exclusion config — shared with
+            // `recrawl`, see `#classifyExcludedNovelUrls`. `effectiveConfig`
+            // is also used below to build `baseConfig`.
             const effectiveConfig = { ...archived, ...cleanObject(options) };
-            const excludes = normalizeToArray(effectiveConfig.excludes);
-            const excludeUrls = [
-                ...DEFAULT_EXCLUDED_EXTERNAL_URLS,
-                ...normalizeToArray(effectiveConfig.excludeUrls),
-            ];
-            const excludedNovelUrls = [];
-            const importableNovelUrls = [];
-            for (const url of novelUrls) {
-                if (shouldSkipUrl({ url, excludes, excludeUrls, options: effectiveConfig })) {
-                    excludedNovelUrls.push(url);
-                }
-                else {
-                    importableNovelUrls.push(url);
-                }
-            }
-            if (excludedNovelUrls.length > 0) {
-                log('[inventory] %d URL(s) recorded as skipped (matched excludes / excludeUrls)', excludedNovelUrls.length);
-            }
+            const { excludedNovelUrls, importableNovelUrls } = CrawlerOrchestrator.#classifyExcludedNovelUrls(novelUrls, effectiveConfig);
             if (novelUrls.length === 0) {
                 // Nothing to do — release the archive cleanly without taking a
                 // backup. The orchestrator returned here is empty; the caller
@@ -997,89 +1364,17 @@ export class CrawlerOrchestrator extends EventEmitter {
             // clause). This flag steers the catch below.
             let ingestionComplete = false;
             try {
-                // Classify importable novel URLs by URL-extension heuristic (no I/O).
-                // Source file lists come from `ls` on the doc-root, so the
-                // extension reflects the real file type — a HEAD pre-flight
-                // here would be pure wasted I/O. Edge cases:
-                //
-                // - `.html` returning 404 / 200: the normal crawler HEAD/GET
-                //   path absorbs this because every HTML-classified URL is
-                //   fed through the dealer and gets its real HEAD/GET there.
-                //
-                // - Extensionless API endpoints (e.g. `/api/foo`) that the
-                //   server returns as `text/html`: `isLikelyHtmlUrl` accepts
-                //   them as HTML so the dealer's render path runs — the
-                //   real content-type wins downstream.
-                //
-                // - `.aspx` / `.do` / `.jsp` / other server-handler
-                //   extensions that the heuristic does NOT recognise as
-                //   HTML: these are classified as non-HTML here, recorded
-                //   as `resources` rows with all-null metadata, and never
-                //   get a HEAD/GET probe. The accepted trade-off for
-                //   `--inventory`'s "list of static-looking server files"
-                //   contract; sites that mix server-handlers into the
-                //   inventory list will need a follow-up `--retry-failed`
-                //   pass (or a re-`--inventory` with the corrected list)
-                //   to populate metadata.
-                //
-                // non-HTML rows are recorded with null status/content-type
-                // which is sufficient for `listUnusedResources` (referrer
-                // count = 0) but means downstream consumers must treat
-                // null as "not probed" rather than "failed".
-                const rawHtmlSeeds = [];
-                const nonHtmlSeeds = [];
-                for (const url of importableNovelUrls) {
-                    if (isLikelyHtmlUrl(url)) {
-                        rawHtmlSeeds.push(url);
-                    }
-                    else {
-                        nonHtmlSeeds.push(url);
-                    }
-                }
-                // Dedup HTML seeds by `protocolAgnosticKey` so an inventory
-                // list that mixes `http://` and `https://` for the same
-                // origin does not produce two `pages` rows that the dealer
-                // later collapses to one — the loser would otherwise stay
-                // `scraped=0, source='inventory-seed'` forever and look like
-                // a real recovery candidate on `--resume`. `getExistingPageUrls`
-                // keys on the full URL (with protocol), so it cannot catch
-                // the cross-scheme duplicate; this is the dedup boundary.
-                const seenKeys = new Set();
-                const htmlSeeds = [];
-                for (const url of rawHtmlSeeds) {
-                    const key = protocolAgnosticKey(url.withoutHashAndAuth);
-                    if (seenKeys.has(key)) {
-                        continue;
-                    }
-                    seenKeys.add(key);
-                    htmlSeeds.push(url);
-                }
-                // Bulk-record non-HTML novel URLs in `resources` as
-                // `source='inventory-seed'` placeholders. A
-                // per-URL `await setResources(...)` loop would spend minutes
-                // inside the `.bak`-protected window on large inventory
-                // lists; the chunked bulk path collapses N round-trips
-                // to N/500.
-                setupProgress?.onPhase?.(PHASE_RECORDING_NON_HTML);
-                await archive.insertInventoryResources(nonHtmlSeeds);
-                // Pre-insert HTML seeds as `scraped = 0`,
-                // `source = 'inventory-seed'` placeholders *before* the
-                // scrape phase, so a Ctrl+C between here and `setPage`
-                // cannot lose the URL. The strict-pending set picks
-                // these rows up on the next `--resume` via the
-                // `OR p.source != 'crawled'` clause.
-                setupProgress?.onPhase?.(PHASE_RECORDING_HTML_SEEDS);
-                await archive.insertInventorySeeds(htmlSeeds);
-                // Record exclude-matched novel URLs as terminal skipped pages
-                // (`is_skipped=1`, `skip_reason='excluded'`,
-                // `source='inventory-seed'`) — the same end state the normal
-                // crawl's fetch-time gate produces for link-discovered
-                // excluded URLs, so the archive looks identical no matter
-                // how the URL was discovered. Inside the `.bak` window for
-                // the same all-or-nothing reason as the seed inserts above.
-                setupProgress?.onPhase?.(PHASE_RECORDING_EXCLUDED);
-                await archive.insertInventorySkippedPages(excludedNovelUrls);
-                log('[inventory] %d HTML seed(s), %d non-HTML resource(s), %d skipped page(s) recorded', htmlSeeds.length, nonHtmlSeeds.length, excludedNovelUrls.length);
+                // Classify, dedup, and bulk-record the novel URLs — shared
+                // with `recrawl`, see `#ingestNovelSeeds`.
+                const { htmlSeeds, nonHtmlSeeds } = await CrawlerOrchestrator.#ingestNovelSeeds({
+                    archive,
+                    importableNovelUrls,
+                    excludedNovelUrls,
+                    setupProgress,
+                    phaseRecordingNonHtml: PHASE_RECORDING_NON_HTML,
+                    phaseRecordingHtmlSeeds: PHASE_RECORDING_HTML_SEEDS,
+                    phaseRecordingExcluded: PHASE_RECORDING_EXCLUDED,
+                });
                 // Audit row is written *inside* the `.bak` window: a libsql
                 // hiccup or transient lock on the INSERT aborts the ingestion
                 // and the `.bak` restore wipes the pre-inserted seeds too,
@@ -1087,8 +1382,8 @@ export class CrawlerOrchestrator extends EventEmitter {
                 // the ingestion boundary. Audit failures are deliberately
                 // NOT swallowed — inside the `.bak` window a restore is
                 // safe and useful (see
-                // {@link CrawlerOrchestrator.#writeInventoryRunRow}).
-                await CrawlerOrchestrator.#writeInventoryRunRow(archive, {
+                // {@link CrawlerOrchestrator.#writeListReconcileRunRow}).
+                await CrawlerOrchestrator.#writeListReconcileRunRow(archive, {
                     inventoryUrlsCount: inventoryUrls.length,
                     htmlSeedsCount: htmlSeeds.length,
                     nonHtmlCount: nonHtmlSeeds.length,
@@ -1129,13 +1424,25 @@ export class CrawlerOrchestrator extends EventEmitter {
                     // Seed the sticky set from prior sessions' confirmed traps
                     // so `--inventory` does not pay the cost of
                     // re-discovering them (see `DedupeCapTracker`'s
-                    // constructor JSDoc). Scoped to this branch only,
-                    // matching `#preloadDnsBurnedHostCache`'s scoping below —
-                    // the fallback (non-HTML-only) branch never calls
+                    // constructor JSDoc), and replay every not-yet-capped
+                    // shape's prior observations so its counter does not
+                    // restart at 0 (see `#preloadDedupeCapObservations`).
+                    // Scoped to this branch only, matching
+                    // `#preloadDnsBurnedHostCache`'s scoping below — the
+                    // fallback (non-HTML-only) branch never calls
                     // `orchestrator.crawling(...)`, so the tracker is never
-                    // consulted there.
-                    orchestratorOptions.preloadedStickyShapeKeys =
-                        await archive.listDedupeCapShapeKeys();
+                    // consulted there. Silent (no `onPhase`/`onProgress`) —
+                    // matching the shape-key read's existing silent
+                    // behaviour here, unlike `append`/`retryFailed`/`resume`,
+                    // which already dedicate a phase to it. Independent reads
+                    // — run concurrently (see `append`'s identical pair).
+                    [
+                        orchestratorOptions.preloadedStickyShapeKeys,
+                        orchestratorOptions.preloadedDedupeObservations,
+                    ] = await Promise.all([
+                        archive.listDedupeCapShapeKeys(),
+                        CrawlerOrchestrator.#preloadDedupeCapObservations(archive, orchestratorOptions.dedupeCap ?? null),
+                    ]);
                     const orchestrator = new CrawlerOrchestrator(archive, orchestratorOptions);
                     // Re-read pending *after* the pre-insert so the strict-
                     // pending set includes the freshly inserted
@@ -1145,7 +1452,14 @@ export class CrawlerOrchestrator extends EventEmitter {
                     // pending set alone (see retryFailed's
                     // `crawling([], { recursive })` invocation).
                     setupProgress?.onPhase?.(PHASE_LOADING_CRAWL_STATE_POST);
-                    const { scraped: scrapedAfter, pending: pendingAfter } = await archive.getCrawlingState();
+                    // `archived.fromList` is rejected above, so no
+                    // root-exclusion is needed here (see the list-mode
+                    // `resume`/auto-retry paths' comments for why it matters
+                    // there). No `= []` default needed: `pendingMetadataOnlyAfter`
+                    // is only ever passed straight through to `Crawler#resume()`,
+                    // whose own `metadataOnlyUrls` parameter already defaults
+                    // `undefined` to `[]`.
+                    const { scraped: scrapedAfter, pending: pendingAfter, pendingMetadataOnly: pendingMetadataOnlyAfter, } = await archive.getCrawlingState();
                     setupProgress?.onPhase?.(PHASE_LOADING_RESOURCES);
                     const resources = await archive.getResourceUrlList(setupProgress?.onChunkProgress);
                     // Pre-existing rendered HTML page count seeds the
@@ -1157,7 +1471,7 @@ export class CrawlerOrchestrator extends EventEmitter {
                     setupProgress?.onPhase?.(PHASE_LOADING_SCRAPED_COUNT);
                     const pagesScrapedOffset = await archive.getScrapedHtmlPageCount();
                     setupProgress?.onPhase?.(PHASE_RESTORING_CRAWL_STATE);
-                    orchestrator.#crawler.resume(pendingAfter, scrapedAfter, resources, pagesScrapedOffset);
+                    orchestrator.#crawler.resume(pendingAfter, scrapedAfter, resources, pagesScrapedOffset, pendingMetadataOnlyAfter);
                     if (initializedCallback) {
                         await initializedCallback(orchestrator, baseConfig);
                     }
@@ -1165,7 +1479,7 @@ export class CrawlerOrchestrator extends EventEmitter {
                     log('Archive %s', absFilePath);
                     log('HTML seeds %O', htmlSeeds.map((u) => u.href));
                     await CrawlerOrchestrator.#preloadDnsBurnedHostCache(archive);
-                    await orchestrator.crawling([], { recursive: true });
+                    await orchestrator.#crawlUntilPendingClears([], { recursive: true });
                     CrawlerOrchestrator.#finalizeCrawlSession(orchestrator);
                     await orchestrator.#setUrlOrder();
                     return orchestrator;
@@ -1181,28 +1495,33 @@ export class CrawlerOrchestrator extends EventEmitter {
             }
             catch (error) {
                 if (ingestionComplete) {
-                    // Scrape phase failed; the pre-inserted seeds + audit
-                    // row are durable inside `tmpDir/db.sqlite` but not yet
-                    // on disk as a `.nitpicker` tar. The outer catch below
-                    // runs `archive.close()`, which sees the original
+                    // Scrape phase failed — either the auto-retry loop
+                    // (`#crawlUntilPendingClears`, issue #350) gave up with
+                    // pages still pending, or some other exception. Either
+                    // way the pre-inserted seeds + audit row are durable
+                    // inside `tmpDir/db.sqlite` but must NOT be packaged: a
+                    // `.nitpicker` on disk must imply `pending === 0` (see
+                    // that method's JSDoc). The outer catch below runs
+                    // `archive.close()`, which sees the original
                     // (pre-inventory) `.nitpicker` already on disk and
                     // would just `remove(tmpDir)` — silently wiping every
                     // `inventory-seed` row and the audit row.
                     //
-                    // Persist the ingested state ourselves before letting
-                    // the outer catch unwind, then re-throw so the operator
-                    // learns about the scrape failure (and can recover via
-                    // `crawl --resume <archive>`). `releaseHandle` shares
-                    // the orchestrator's `#closeOnce` guard, so the outer
-                    // catch's `close()` becomes a no-op for the destructive
-                    // step and only runs `releaseLock` cleanup.
+                    // Release the handle ourselves (leaving tmpDir intact)
+                    // before letting the outer catch unwind, then re-throw
+                    // so the operator learns about the scrape failure and
+                    // can recover via `crawl --resume <stub>`. `releaseHandle`
+                    // shares the orchestrator's `#closeOnce` guard, so the
+                    // outer catch's `close()` becomes a no-op for the
+                    // destructive step and only runs `releaseLock` cleanup
+                    // — a no-op too when `#crawlUntilPendingClears` already
+                    // released it itself before throwing.
                     try {
-                        setupProgress?.onPhase?.(RECOVERY_PERSIST_INGESTED_STATE);
-                        await archive.write();
+                        setupProgress?.onPhase?.(RECOVERY_LEAVE_STATE_FOR_RESUME);
                         await archive.releaseHandle();
                     }
                     catch (persistError) {
-                        throw new AggregateError([error, persistError], 'inventory scrape phase failed AND persisting the ingested state to disk also failed. The archive may be in an inconsistent state — check tmpDir.');
+                        throw new AggregateError([error, persistError], 'inventory scrape phase failed AND releasing the archive handle also failed. The archive may be in an inconsistent state — check tmpDir.');
                     }
                     throw error;
                 }
@@ -1222,6 +1541,481 @@ export class CrawlerOrchestrator extends EventEmitter {
             throw error;
         }
     }
+    /**
+     * Re-fetch pages named by an operator-supplied URL list, importing any
+     * URL the archive does not yet track as a new inventory seed.
+     *
+     * `recrawl` is `retryFailed`'s un-scrape combined with `inventory`'s
+     * novel-URL ingestion, run inside one `.bak`-protected window: URLs in
+     * `recrawlUrls` that already exist as `content_items` rows are reset back
+     * to pending via {@link Archive.resetPagesByUrls} (see that method for the
+     * conservative exclusion rules — redirect sources, intentionally-skipped
+     * pages, and external pages are matched but never reset), while URLs the
+     * archive has never seen are ingested exactly as `inventory` does (see
+     * {@link CrawlerOrchestrator.inventory}'s JSDoc for that half's contract).
+     * Existing *resources* matched by the list are neither resettable nor
+     * novel — `resource_items` is first-write-wins (a known deviation, see
+     * ARCHITECTURE.md), so a resource re-fetch would not update anything; the
+     * function reports how many list entries fell into this bucket via
+     * `setupProgress.onLog` without acting on them.
+     *
+     * Unlike `inventory`, whose sole early-return condition is "no novel
+     * URLs", `recrawl` also has existing pages to act on — the `.bak` is
+     * skipped only when BOTH `existingPageUrls` (reset candidates) AND
+     * `novelUrls` (ingestion candidates) are empty.
+     *
+     * **Strict-pending gap**: `getCrawlingState()`'s pending set only includes
+     * a `scraped = 0` row that is either anchor-referenced or explicitly
+     * labelled (see that function's JSDoc). When `recrawlUrls` contains pages
+     * that link to each other, resetting one page also deletes its outgoing
+     * `anchor_edges` — so a `source = 'crawled'` sibling that was reset in the
+     * same pass can lose its only anchor referrer and fall out of the strict
+     * pending set, silently skipping its re-fetch. `retryFailed` never hits
+     * this because a failed page's referrers are not themselves reset. The
+     * fix: every URL `Archive.resetPagesByUrls` actually reset is merged into
+     * the pending list handed to `Crawler#resume` regardless of what the
+     * strict scan finds, deduplicated by `LinkList.add`'s `protocolAgnosticKey`
+     * check. A Ctrl+C between the reset and the scrape phase loses this
+     * synthetic merge (it lives only in memory) — `crawl --resume` recovers
+     * whatever the strict-pending scan finds on its own, and re-running
+     * `--recrawl` with the same list recovers the rest, matching the
+     * "un-picked seeds" recovery contract `getCrawlingState`'s JSDoc already
+     * documents for `inventory`.
+     *
+     * **Stale analyze findings**: resetting a page deletes its
+     * `analysis_violations` rows (see {@link resetPagesByUrls}'s JSDoc) so a
+     * re-fetched page never shows findings from HTML that no longer exists,
+     * but other `analyze` outputs (e.g. Discrepancies plugin reports) are not
+     * page-scoped and cannot be selectively invalidated. When at least one
+     * page was reset, a `crawlSessionNotice` is emitted after the crawl
+     * completes recommending `analyze` be re-run before the next `report`.
+     * @param archivePath - Absolute or relative path to the existing `.nitpicker`.
+     * @param recrawlUrls - URLs to match against the archive (existing pages
+     *   are reset; unknown URLs are ingested as new inventory seeds).
+     * @param options - Optional config overrides applied on top of the archived config.
+     * @param initializedCallback - Optional callback invoked after initialization but before crawling resumes.
+     * @param source - The CLI's already-read URL list source bytes, archived
+     *   for audit purposes — see {@link InventorySource}. `null` for
+     *   programmatic callers with no source file.
+     * @param setupProgress - Optional progress callbacks for the setup phase
+     *   (untar, `.bak` copy, URL classification, reset, seed ingestion, state
+     *   rebuild) that runs before `initializedCallback` — see
+     *   {@link SetupProgressCallbacks} for why this can't go through the
+     *   orchestrator's event emitter (issue #294).
+     * @returns The orchestrator instance after the recrawl completes.
+     * @throws {Error} When `recrawlUrls` is empty or the archive is in list mode.
+     * @throws {PendingUrlsRemainError} When the crawl session ends with pages still pending after exhausting auto-retry.
+     */
+    static async recrawl(archivePath, recrawlUrls, options, initializedCallback, source = null, setupProgress) {
+        const [PHASE_EXTRACTING, PHASE_LOADING_CONFIG, PHASE_LOADING_CRAWL_STATE_PRE, PHASE_CHECKING_KNOWN_URLS, PHASE_BACKING_UP, PHASE_RESETTING_MATCHED, PHASE_RECORDING_NON_HTML, PHASE_RECORDING_HTML_SEEDS, PHASE_RECORDING_EXCLUDED, PHASE_LOADING_CRAWL_STATE_POST, PHASE_LOADING_RESOURCES, PHASE_LOADING_SCRAPED_COUNT, PHASE_RESTORING_CRAWL_STATE,] = RECRAWL_SETUP_PHASES;
+        if (recrawlUrls.length === 0) {
+            throw new Error('recrawl: URL list is empty');
+        }
+        const cwd = options?.cwd ?? process.cwd();
+        const absFilePath = path.isAbsolute(archivePath)
+            ? archivePath
+            : path.resolve(cwd, archivePath);
+        // See `ArchiveOpenOptions.openPluginData` for why this must be `true`
+        // on every writer path that calls `write()`.
+        setupProgress?.onPhase?.(PHASE_EXTRACTING);
+        const archive = await Archive.open({
+            filePath: absFilePath,
+            cwd,
+            openPluginData: true,
+            onExtractProgress: setupProgress?.onExtractProgress,
+            onLog: setupProgress?.onLog,
+        });
+        try {
+            setupProgress?.onPhase?.(PHASE_LOADING_CONFIG);
+            const archived = await archive.getConfig();
+            if (archived.fromList) {
+                throw new Error('Cannot recrawl a list-mode archive: this archive was created with --list/--list-file and contains metadata-only pages. Create a fresh archive instead.');
+            }
+            // Stamped for `Archive.resume` (issue #350) — same rationale as
+            // `inventory`'s identical call.
+            await archive.updateConfig(buildCreatedCwdPatch(cwd));
+            setupProgress?.onPhase?.(PHASE_LOADING_CRAWL_STATE_PRE);
+            const { pending } = await archive.getCrawlingState();
+            if (pending.length > 0) {
+                // Same rationale as `inventory`'s identical warning — routed
+                // through `setupProgress.onLog`, not a bare `console.warn`,
+                // since the `'Loading crawl state'` row is active here.
+                const message = `recrawl: archive has ${pending.length} pending URLs from a previous crawl. Proceeding — crawled-wins priority keeps their labels stable. Consider \`--resume\` first if you want the prior work finalized.`;
+                if (setupProgress?.onLog) {
+                    setupProgress.onLog(message);
+                }
+                else {
+                    // eslint-disable-next-line no-console -- --silent has no TaskList row to report through
+                    console.warn(message);
+                }
+            }
+            // Archive the exact source bytes before scope classification —
+            // same rationale as `inventory`'s identical call.
+            if (source) {
+                await archive.saveInventorySourceList(source.sha256, source.bytes);
+            }
+            setupProgress?.onPhase?.(PHASE_CHECKING_KNOWN_URLS);
+            const { outOfScope, existingPageUrls, existingResourceUrls, novelUrls } = await CrawlerOrchestrator.#classifyInventoryCandidateUrls(recrawlUrls, archived, archive);
+            if (existingResourceUrls.length > 0) {
+                const message = `recrawl: ${existingResourceUrls.length} URL(s) matched existing resources — not re-fetched (resource rows are first-write-wins; re-fetching would not update them).`;
+                if (setupProgress?.onLog) {
+                    setupProgress.onLog(message);
+                }
+                else {
+                    // eslint-disable-next-line no-console -- --silent has no TaskList row to report through
+                    console.warn(message);
+                }
+            }
+            const effectiveConfig = { ...archived, ...cleanObject(options) };
+            const { excludedNovelUrls, importableNovelUrls } = CrawlerOrchestrator.#classifyExcludedNovelUrls(novelUrls, effectiveConfig);
+            if (existingPageUrls.length === 0 && novelUrls.length === 0) {
+                // Nothing to do — release the archive cleanly without taking a
+                // backup, mirroring `inventory`'s zero-novel early return.
+                const orchestrator = new CrawlerOrchestrator(archive, effectiveConfig);
+                if (initializedCallback) {
+                    await initializedCallback(orchestrator, effectiveConfig);
+                }
+                return orchestrator;
+            }
+            const backupPath = absFilePath + '.bak';
+            setupProgress?.onPhase?.(PHASE_BACKING_UP);
+            await copyFileWithProgress(absFilePath, backupPath, setupProgress?.onCopyProgress);
+            let ingestionComplete = false;
+            try {
+                setupProgress?.onPhase?.(PHASE_RESETTING_MATCHED);
+                const resetResult = await archive.resetPagesByUrls(existingPageUrls, setupProgress?.onChunkProgress);
+                const excludedTotal = resetResult.excludedRedirects.length +
+                    resetResult.excludedSkipped.length +
+                    resetResult.excludedExternal.length;
+                // `existingPageUrls` (from `getExistingPageUrls`) matches by URL
+                // alone, regardless of `scraped` — it can include rows still
+                // pending from an interrupted previous session. Those rows are
+                // absent from every `resetPagesByUrls` array (see that
+                // function's JSDoc: "already pending, nothing to reset"), so
+                // the three counts below alone would not sum back to
+                // `existingPageUrls.length` and the message would look like
+                // pages vanished unexplained. Naming this remainder keeps the
+                // arithmetic honest for an operator auditing the summary.
+                const alreadyPendingCount = existingPageUrls.length - resetResult.resetUrls.length - excludedTotal;
+                const summaryMessage = `recrawl: matched ${existingPageUrls.length} existing page(s) — reset ${resetResult.resetUrls.length}, excluded ${excludedTotal} (${resetResult.excludedRedirects.length} redirect source(s), ${resetResult.excludedSkipped.length} intentionally-skipped, ${resetResult.excludedExternal.length} external), already pending ${alreadyPendingCount}.`;
+                if (setupProgress?.onLog) {
+                    setupProgress.onLog(summaryMessage);
+                }
+                else {
+                    // eslint-disable-next-line no-console -- --silent has no TaskList row to report through
+                    console.warn(summaryMessage);
+                }
+                const { htmlSeeds, nonHtmlSeeds } = await CrawlerOrchestrator.#ingestNovelSeeds({
+                    archive,
+                    importableNovelUrls,
+                    excludedNovelUrls,
+                    setupProgress,
+                    phaseRecordingNonHtml: PHASE_RECORDING_NON_HTML,
+                    phaseRecordingHtmlSeeds: PHASE_RECORDING_HTML_SEEDS,
+                    phaseRecordingExcluded: PHASE_RECORDING_EXCLUDED,
+                });
+                // Audit row is written *inside* the `.bak` window — same
+                // all-or-nothing rationale as `inventory`'s identical write.
+                await CrawlerOrchestrator.#writeListReconcileRunRow(archive, {
+                    inventoryUrlsCount: recrawlUrls.length,
+                    htmlSeedsCount: htmlSeeds.length,
+                    nonHtmlCount: nonHtmlSeeds.length,
+                    outOfScope,
+                    excludeSkipped: excludedNovelUrls.length,
+                    sourceFileSha256: source?.sha256 ?? null,
+                    invalidSkipped: source?.invalidLineCount ?? null,
+                    listLabelPrefix: 'recrawl',
+                    notes: `Reset ${resetResult.resetUrls.length} existing page(s) for re-fetch`,
+                });
+                ingestionComplete = true;
+                await ignoreEnoent(unlinkFile(backupPath));
+                const baseConfig = {
+                    ...effectiveConfig,
+                    recursive: true,
+                    fromList: false,
+                };
+                const seedSet = new Set(htmlSeeds.map((u) => u.withoutHashAndAuth));
+                const orchestratorOptions = {
+                    ...baseConfig,
+                    inventoryMode: { seedUrls: seedSet },
+                };
+                if (resetResult.resetUrls.length > 0 || htmlSeeds.length > 0) {
+                    // Same rationale as `inventory`'s identical pair of
+                    // preload calls (sticky shapes + prior observation
+                    // replay, run concurrently), silent for the same reason
+                    // — see that method's comment.
+                    [
+                        orchestratorOptions.preloadedStickyShapeKeys,
+                        orchestratorOptions.preloadedDedupeObservations,
+                    ] = await Promise.all([
+                        archive.listDedupeCapShapeKeys(),
+                        CrawlerOrchestrator.#preloadDedupeCapObservations(archive, orchestratorOptions.dedupeCap ?? null),
+                    ]);
+                    const orchestrator = new CrawlerOrchestrator(archive, orchestratorOptions);
+                    setupProgress?.onPhase?.(PHASE_LOADING_CRAWL_STATE_POST);
+                    // `archived.fromList` is rejected above, so no
+                    // root-exclusion is needed here (see the list-mode
+                    // `resume`/auto-retry paths' comments for why it matters
+                    // there). `pendingMetadataOnlyAfter` is not merged with
+                    // `resetResult.resetUrls` the way `pending` is below — a
+                    // reset row absent from the strict-pending set defaults
+                    // to a full re-scrape, the safe direction for a
+                    // user-requested recrawl. No `= []` default needed: it is
+                    // only ever passed straight through to `Crawler#resume()`,
+                    // whose own `metadataOnlyUrls` parameter already defaults
+                    // `undefined` to `[]`.
+                    const { scraped: scrapedAfter, pending: pendingAfter, pendingMetadataOnly: pendingMetadataOnlyAfter, } = await archive.getCrawlingState();
+                    // Merge the reset URLs into the pending set explicitly —
+                    // see this method's "Strict-pending gap" JSDoc section.
+                    // Deduped by `LinkList.add`'s own key check, so a URL the
+                    // strict scan already found is harmless to repeat here.
+                    const pendingWithReset = [
+                        ...new Set([...pendingAfter, ...resetResult.resetUrls]),
+                    ];
+                    setupProgress?.onPhase?.(PHASE_LOADING_RESOURCES);
+                    const resources = await archive.getResourceUrlList(setupProgress?.onChunkProgress);
+                    setupProgress?.onPhase?.(PHASE_LOADING_SCRAPED_COUNT);
+                    const pagesScrapedOffset = await archive.getScrapedHtmlPageCount();
+                    setupProgress?.onPhase?.(PHASE_RESTORING_CRAWL_STATE);
+                    orchestrator.#crawler.resume(pendingWithReset, scrapedAfter, resources, pagesScrapedOffset, pendingMetadataOnlyAfter);
+                    if (initializedCallback) {
+                        await initializedCallback(orchestrator, baseConfig);
+                    }
+                    log('Start recrawl');
+                    log('Archive %s', absFilePath);
+                    log('Reset %d page(s), %d new HTML seed(s)', resetResult.resetUrls.length, htmlSeeds.length);
+                    await CrawlerOrchestrator.#preloadDnsBurnedHostCache(archive);
+                    await orchestrator.#crawlUntilPendingClears([], { recursive: true });
+                    CrawlerOrchestrator.#finalizeCrawlSession(orchestrator);
+                    if (resetResult.resetUrls.length > 0) {
+                        void orchestrator.emit('crawlSessionNotice', {
+                            message: `[recrawl] Reset ${resetResult.resetUrls.length} page(s) — run \`analyze\` before \`report\` to refresh their findings.`,
+                        });
+                    }
+                    await orchestrator.#setUrlOrder();
+                    return orchestrator;
+                }
+                // Only non-HTML URLs were imported and nothing was reset —
+                // nothing left to render, but still update sort order.
+                const orchestrator = new CrawlerOrchestrator(archive, orchestratorOptions);
+                if (initializedCallback) {
+                    await initializedCallback(orchestrator, baseConfig);
+                }
+                await orchestrator.#setUrlOrder();
+                return orchestrator;
+            }
+            catch (error) {
+                if (ingestionComplete) {
+                    // Same rationale as `inventory`'s identical catch — see
+                    // that method's comment. Scrape failure here can equally
+                    // be the auto-retry loop (issue #350) giving up.
+                    try {
+                        setupProgress?.onPhase?.(RECOVERY_LEAVE_STATE_FOR_RESUME);
+                        await archive.releaseHandle();
+                    }
+                    catch (persistError) {
+                        throw new AggregateError([error, persistError], 'recrawl scrape phase failed AND releasing the archive handle also failed. The archive may be in an inconsistent state — check tmpDir.');
+                    }
+                    throw error;
+                }
+                try {
+                    setupProgress?.onPhase?.(RECOVERY_RESTORE_FROM_BACKUP);
+                    await copyFileWithProgress(backupPath, absFilePath, setupProgress?.onCopyProgress);
+                    await ignoreEnoent(unlinkFile(backupPath));
+                }
+                catch (restoreError) {
+                    throw new AggregateError([error, restoreError], `recrawl failed AND restore from backup failed. Original archive backup is left at: ${backupPath}`);
+                }
+                throw error;
+            }
+        }
+        catch (error) {
+            await archive.close().catch(() => { });
+            throw error;
+        }
+    }
+    /**
+     * Shared first-stage classification for `inventory` and `recrawl`: parse
+     * the candidate URLs, split them by the archived scope map into
+     * in-scope/out-of-scope, then split the in-scope set into URLs already
+     * represented in the archive (as a page or a resource) vs. novel URLs the
+     * archive has never seen. Comparison key is `withoutHashAndAuth` to
+     * mirror what `resolveContentItemId` / `insertResource` actually store.
+     *
+     * The two existing-URL reads run concurrently via `Promise.all` — halves
+     * the wait on large archives where each `WHERE url IN (?)` chunk costs
+     * real I/O.
+     * @param rawUrls - The operator-supplied URL list, unparsed.
+     * @param archived - The archive's persisted config (`roots` defines scope).
+     * @param archive - The opened archive to query for existing URLs.
+     * @returns `outOfScope` (count dropped by the scope filter),
+     *   `existingPageUrls` / `existingResourceUrls` (URLs already known, by
+     *   kind), and `novelUrls` (parsed, in-scope URLs matching neither).
+     */
+    static async #classifyInventoryCandidateUrls(rawUrls, archived, archive) {
+        const parsedAll = sortUrl(rawUrls, archived);
+        const scopeMap = new Map();
+        for (const raw of archived.roots) {
+            const parsed = parseUrl(raw, archived);
+            if (!parsed)
+                continue;
+            const existing = scopeMap.get(parsed.hostname) ?? [];
+            scopeMap.set(parsed.hostname, [...existing, parsed]);
+        }
+        const inScope = [];
+        let outOfScope = 0;
+        for (const url of parsedAll) {
+            if (findScopeEntry(url, scopeMap, archived) === null) {
+                outOfScope++;
+            }
+            else {
+                inScope.push(url);
+            }
+        }
+        if (outOfScope > 0) {
+            log('[ingest] %d URL(s) skipped (outside archived scope: %O)', outOfScope, archived.roots);
+        }
+        const candidateUrls = inScope.map((u) => u.withoutHashAndAuth);
+        const [existingPageUrls, existingResourceUrls] = await Promise.all([
+            archive.getExistingPageUrls(candidateUrls),
+            archive.getExistingResourceUrls(candidateUrls),
+        ]);
+        const existingPageUrlSet = new Set(existingPageUrls);
+        const existingResourceUrlSet = new Set(existingResourceUrls);
+        const novelUrls = inScope.filter((u) => {
+            const key = u.withoutHashAndAuth;
+            return !existingPageUrlSet.has(key) && !existingResourceUrlSet.has(key);
+        });
+        log('[ingest] %d in-scope, %d already in archive, %d new', inScope.length, existingPageUrlSet.size + existingResourceUrlSet.size, novelUrls.length);
+        return { outOfScope, existingPageUrls, existingResourceUrls, novelUrls };
+    }
+    /**
+     * Shared second-stage classification for `inventory` and `recrawl`:
+     * splits novel URLs on the exclusion config BEFORE the HTML/non-HTML
+     * classification, so an exclude-matched URL is recorded as a terminal
+     * skipped page instead of being imported (issue #260).
+     *
+     * The inputs mirror the scrape phase's fetch-time gate (`shouldSkipUrl`
+     * in `crawler.ts` fed by the constructor's merge): archived config
+     * overlaid with this run's overrides, and `DEFAULT_EXCLUDED_EXTERNAL_URLS`
+     * merged ahead of the user's prefixes — classification and gate must
+     * never disagree about the same URL. Running this AFTER the known-URL
+     * filter (`#classifyInventoryCandidateUrls`) is deliberate: a previously
+     * crawled row that newly matches the exclusion config stays untouched
+     * (crawled-wins), matching how `getExistingPageUrls` shields known rows
+     * from re-labelling. `excludeKeywords` is deliberately absent: it matches
+     * rendered page content, which a URL list does not have — HTML seeds
+     * still get it at render time via the browser verdict.
+     * @param novelUrls - URLs not yet represented in the archive, from
+     *   `#classifyInventoryCandidateUrls`.
+     * @param effectiveConfig - The archived config overlaid with this run's
+     *   overrides (the same merge the caller uses to build its own config).
+     * @returns `excludedNovelUrls` (matched `excludes`/`excludeUrls`, to be
+     *   recorded as terminal skipped pages) and `importableNovelUrls` (the rest).
+     */
+    static #classifyExcludedNovelUrls(novelUrls, effectiveConfig) {
+        const excludes = normalizeToArray(effectiveConfig.excludes);
+        const excludeUrls = [
+            ...DEFAULT_EXCLUDED_EXTERNAL_URLS,
+            ...normalizeToArray(effectiveConfig.excludeUrls),
+        ];
+        const excludedNovelUrls = [];
+        const importableNovelUrls = [];
+        for (const url of novelUrls) {
+            if (shouldSkipUrl({ url, excludes, excludeUrls, options: effectiveConfig })) {
+                excludedNovelUrls.push(url);
+            }
+            else {
+                importableNovelUrls.push(url);
+            }
+        }
+        if (excludedNovelUrls.length > 0) {
+            log('[ingest] %d URL(s) recorded as skipped (matched excludes / excludeUrls)', excludedNovelUrls.length);
+        }
+        return { excludedNovelUrls, importableNovelUrls };
+    }
+    /**
+     * Shared third-stage ingestion for `inventory` and `recrawl`: classifies
+     * importable novel URLs by URL-extension heuristic (no I/O), dedups HTML
+     * seeds, and bulk-records both kinds into the archive.
+     *
+     * Source file lists come from `ls` on the doc-root, so the extension
+     * reflects the real file type — a HEAD pre-flight here would be pure
+     * wasted I/O. Edge cases:
+     *
+     * - `.html` returning 404 / 200: the normal crawler HEAD/GET path absorbs
+     *   this because every HTML-classified URL is fed through the dealer and
+     *   gets its real HEAD/GET there.
+     * - Extensionless API endpoints (e.g. `/api/foo`) that the server returns
+     *   as `text/html`: `isLikelyHtmlUrl` accepts them as HTML so the
+     *   dealer's render path runs — the real content-type wins downstream.
+     * - `.aspx` / `.do` / `.jsp` / other server-handler extensions the
+     *   heuristic does NOT recognise as HTML: classified as non-HTML,
+     *   recorded as `resources` rows with all-null metadata, never get a
+     *   HEAD/GET probe. Sites that mix server-handlers into the list need a
+     *   follow-up `--retry-failed` pass (or a re-run with the corrected list)
+     *   to populate metadata.
+     *
+     * HTML seeds are deduped by `protocolAgnosticKey` so a list mixing
+     * `http://` and `https://` for the same origin does not produce two rows
+     * that the dealer later collapses to one — the loser would otherwise stay
+     * `scraped=0, source='inventory-seed'` forever and look like a real
+     * recovery candidate on `--resume`. `getExistingPageUrls` keys on the
+     * full URL (with protocol), so it cannot catch the cross-scheme
+     * duplicate; this is the dedup boundary.
+     *
+     * Non-HTML URLs are bulk-recorded via `insertInventoryResources` — a
+     * per-URL loop would spend minutes inside the `.bak`-protected window on
+     * a large list; the chunked bulk path collapses N round-trips to N/500.
+     * HTML seeds are pre-inserted as `scraped = 0`, `source =
+     * 'inventory-seed'` placeholders *before* the scrape phase, so a Ctrl+C
+     * between here and `setPage` cannot lose the URL — the strict-pending set
+     * picks these rows up on the next `--resume` via the `OR p.source !=
+     * 'crawled'` clause. Exclude-matched novel URLs are recorded as terminal
+     * skipped pages (`is_skipped=1`, `skip_reason='excluded'`,
+     * `source='inventory-seed'`) — the same end state the normal crawl's
+     * fetch-time gate produces for link-discovered excluded URLs.
+     * @param options - Named parameters (4+ values).
+     * @param options.archive - The opened archive to write into.
+     * @param options.importableNovelUrls - Novel URLs not matched by excludes.
+     * @param options.excludedNovelUrls - Novel URLs matched by excludes, from `#classifyExcludedNovelUrls`.
+     * @param options.setupProgress - Optional setup progress callbacks.
+     * @param options.phaseRecordingNonHtml - The `onPhase` label to announce before recording non-HTML resources.
+     * @param options.phaseRecordingHtmlSeeds - The `onPhase` label to announce before recording HTML seed pages.
+     * @param options.phaseRecordingExcluded - The `onPhase` label to announce before recording excluded pages.
+     * @returns `htmlSeeds` and `nonHtmlSeeds` — the deduped, classified novel URLs actually recorded.
+     */
+    static async #ingestNovelSeeds(options) {
+        const { archive, importableNovelUrls, excludedNovelUrls, setupProgress, phaseRecordingNonHtml, phaseRecordingHtmlSeeds, phaseRecordingExcluded, } = options;
+        const rawHtmlSeeds = [];
+        const nonHtmlSeeds = [];
+        for (const url of importableNovelUrls) {
+            if (isLikelyHtmlUrl(url)) {
+                rawHtmlSeeds.push(url);
+            }
+            else {
+                nonHtmlSeeds.push(url);
+            }
+        }
+        const seenKeys = new Set();
+        const htmlSeeds = [];
+        for (const url of rawHtmlSeeds) {
+            const key = protocolAgnosticKey(url.withoutHashAndAuth);
+            if (seenKeys.has(key)) {
+                continue;
+            }
+            seenKeys.add(key);
+            htmlSeeds.push(url);
+        }
+        setupProgress?.onPhase?.(phaseRecordingNonHtml);
+        await archive.insertInventoryResources(nonHtmlSeeds);
+        setupProgress?.onPhase?.(phaseRecordingHtmlSeeds);
+        await archive.insertInventorySeeds(htmlSeeds);
+        setupProgress?.onPhase?.(phaseRecordingExcluded);
+        await archive.insertInventorySkippedPages(excludedNovelUrls);
+        log('[ingest] %d HTML seed(s), %d non-HTML resource(s), %d skipped page(s) recorded', htmlSeeds.length, nonHtmlSeeds.length, excludedNovelUrls.length);
+        return { htmlSeeds, nonHtmlSeeds };
+    }
     /**
      * Re-fetch previously-failed pages in an existing `.nitpicker` archive.
      *
@@ -1245,7 +2039,10 @@ export class CrawlerOrchestrator extends EventEmitter {
      *
      * A `<archive>.bak` is created before any DB mutation and removed on success;
      * if the crawl throws, the backup is restored to keep the original archive
-     * intact.
+     * intact — except when the crawl ends with {@link PendingUrlsRemainError}
+     * (issue #350), where the un-packaged stub itself is the recovery path and
+     * the backup is instead left untouched (deleted, not restored — see
+     * {@link CrawlerOrchestrator.#abandonBackupOnPendingRemains}).
      *
      * List-mode archives (`info.fromList === true`) are rejected for the same
      * reason as {@link CrawlerOrchestrator.append}: their pages are metadata-only.
@@ -1258,6 +2055,7 @@ export class CrawlerOrchestrator extends EventEmitter {
      *   this can't go through the orchestrator's event emitter (issue #294).
      * @returns The orchestrator instance after the retry crawl completes.
      * @throws {Error} When the archive is in list mode or has no parseable roots.
+     * @throws {PendingUrlsRemainError} When the crawl session ends with pages still pending after exhausting auto-retry.
      */
     static async retryFailed(archivePath, options, initializedCallback, setupProgress) {
         const [PHASE_EXTRACTING, PHASE_LOADING_CONFIG, PHASE_BACKING_UP, PHASE_RESETTING_FAILED, PHASE_LOADING_DEDUPE_KEYS, PHASE_LOADING_CRAWL_STATE, PHASE_LOADING_RESOURCES, PHASE_LOADING_SCRAPED_COUNT, PHASE_RESTORING_CRAWL_STATE,] = RETRY_FAILED_SETUP_PHASES;
@@ -1284,6 +2082,9 @@ export class CrawlerOrchestrator extends EventEmitter {
             if (archived.fromList) {
                 throw new Error('Cannot retry a list-mode archive: this archive was created with --list/--list-file and contains metadata-only pages. Create a fresh archive instead.');
             }
+            // Stamped for `Archive.resume` (issue #350) — same rationale as
+            // `inventory`'s identical call.
+            await archive.updateConfig(buildCreatedCwdPatch(cwd));
             const rootsParsed = sortUrl(archived.roots, archived);
             if (rootsParsed.length === 0) {
                 throw new Error('retry: archive has no parseable root URLs');
@@ -1306,32 +2107,60 @@ export class CrawlerOrchestrator extends EventEmitter {
                 log('Reset %d failed page(s)', resetUrls.length);
                 // Seed the sticky set from prior sessions' confirmed traps so
                 // `--retry-failed` does not pay the cost of re-discovering
-                // them (see `DedupeCapTracker`'s constructor JSDoc).
+                // them (see `DedupeCapTracker`'s constructor JSDoc), and
+                // replay every not-yet-capped shape's prior observations so
+                // its counter does not restart at 0 (see
+                // `#preloadDedupeCapObservations`). Runs *after*
+                // `resetFailedPages` above, which already excludes
+                // confirmed-capped-shape failures from the reset — so the
+                // still-`scraped=1` rows this reads back never include a page
+                // this same call is about to re-queue, and a page this call
+                // does reset is correctly absent from the replay (its
+                // `page_meta` row — and therefore its `body_hash` — was just
+                // deleted).
                 setupProgress?.onPhase?.(PHASE_LOADING_DEDUPE_KEYS);
-                const preloadedStickyShapeKeys = await archive.listDedupeCapShapeKeys();
+                // Independent reads — run concurrently (see `append`'s
+                // identical pair).
+                const [preloadedStickyShapeKeys, preloadedDedupeObservations] = await Promise.all([
+                    archive.listDedupeCapShapeKeys(),
+                    CrawlerOrchestrator.#preloadDedupeCapObservations(archive, options?.dedupeCap ?? null, setupProgress?.onChunkProgress),
+                ]);
                 const orchestrator = new CrawlerOrchestrator(archive, {
                     ...config,
                     preloadedStickyShapeKeys,
+                    preloadedDedupeObservations,
                 });
                 setupProgress?.onPhase?.(PHASE_LOADING_CRAWL_STATE);
-                const { scraped, pending } = await archive.getCrawlingState();
+                // `archived.fromList` is rejected above, so this archive's
+                // `recursive` was never `false` for a list-mode reason — no
+                // root-exclusion needed (see the list-mode `resume`/
+                // auto-retry paths' comments for why it matters there). No
+                // `= []` default needed: `pendingMetadataOnly` is only ever
+                // passed straight through to `Crawler#resume()`, whose own
+                // `metadataOnlyUrls` parameter already defaults `undefined`
+                // to `[]`.
+                const { scraped, pending, pendingMetadataOnly } = await archive.getCrawlingState();
                 setupProgress?.onPhase?.(PHASE_LOADING_RESOURCES);
                 const resources = await archive.getResourceUrlList(setupProgress?.onChunkProgress);
                 setupProgress?.onPhase?.(PHASE_LOADING_SCRAPED_COUNT);
                 const pagesScrapedOffset = await archive.getScrapedHtmlPageCount();
                 setupProgress?.onPhase?.(PHASE_RESTORING_CRAWL_STATE);
-                orchestrator.#crawler.resume(pending, scraped, resources, pagesScrapedOffset);
+                orchestrator.#crawler.resume(pending, scraped, resources, pagesScrapedOffset, pendingMetadataOnly);
                 if (initializedCallback) {
                     await initializedCallback(orchestrator, config);
                 }
                 await CrawlerOrchestrator.#preloadDnsBurnedHostCache(archive);
-                await orchestrator.crawling([], { recursive: config.recursive });
+                await orchestrator.#crawlUntilPendingClears([], { recursive: config.recursive });
                 CrawlerOrchestrator.#finalizeCrawlSession(orchestrator);
                 await orchestrator.#setUrlOrder();
                 await ignoreEnoent(unlinkFile(backupPath));
                 return orchestrator;
             }
             catch (error) {
+                if (error instanceof PendingUrlsRemainError) {
+                    await CrawlerOrchestrator.#abandonBackupOnPendingRemains(setupProgress, backupPath);
+                    throw error;
+                }
                 try {
                     setupProgress?.onPhase?.(RECOVERY_RESTORE_FROM_BACKUP);
                     await copyFileWithProgress(backupPath, absFilePath, setupProgress?.onCopyProgress);
@@ -1370,6 +2199,7 @@ export class CrawlerOrchestrator extends EventEmitter {
      *   tmpDir IS the source of truth).
      * @returns A promise that resolves to the CrawlerOrchestrator instance after crawling completes.
      * @throws {Error} If the archived URL is invalid.
+     * @throws {PendingUrlsRemainError} When the crawl session ends with pages still pending after exhausting auto-retry.
      */
     static async resume(stubPath, options, initializedCallback, setupProgress) {
         const [PHASE_RECONNECTING, PHASE_LOADING_CONFIG, PHASE_LOADING_DEDUPE_KEYS, PHASE_LOADING_CRAWL_STATE, PHASE_LOADING_RESOURCES, PHASE_LOADING_SCRAPED_COUNT, PHASE_RESTORING_CRAWL_STATE,] = RESUME_SETUP_PHASES;
@@ -1379,13 +2209,21 @@ export class CrawlerOrchestrator extends EventEmitter {
         const archivedConfig = await archive.getConfig();
         // Seed the sticky set from prior sessions' confirmed traps so
         // `--resume` does not pay the cost of re-discovering them (see
-        // `DedupeCapTracker`'s constructor JSDoc).
+        // `DedupeCapTracker`'s constructor JSDoc), and replay every
+        // not-yet-capped shape's prior observations so its counter does not
+        // restart at 0 (see `#preloadDedupeCapObservations`).
         setupProgress?.onPhase?.(PHASE_LOADING_DEDUPE_KEYS);
-        const preloadedStickyShapeKeys = await archive.listDedupeCapShapeKeys();
+        // Independent reads — run concurrently (see `append`'s identical
+        // pair).
+        const [preloadedStickyShapeKeys, preloadedDedupeObservations] = await Promise.all([
+            archive.listDedupeCapShapeKeys(),
+            CrawlerOrchestrator.#preloadDedupeCapObservations(archive, options?.dedupeCap ?? null, setupProgress?.onChunkProgress),
+        ]);
         const config = {
             ...archivedConfig,
             ...cleanObject(options),
             preloadedStickyShapeKeys,
+            preloadedDedupeObservations,
         };
         const orchestrator = new CrawlerOrchestrator(archive, config);
         const _url = await archive.getUrl();
@@ -1394,13 +2232,31 @@ export class CrawlerOrchestrator extends EventEmitter {
             throw new Error(`URL (${_url}) is invalid`);
         }
         setupProgress?.onPhase?.(PHASE_LOADING_CRAWL_STATE);
-        const { scraped, pending } = await archive.getCrawlingState();
+        const { scraped, pending, pendingMetadataOnly = [], } = await archive.getCrawlingState();
+        // Unlike `append`/`inventory`/`recrawl`/`retryFailed`, this path
+        // resumes ANY archive including list-mode ones (`fromList: true`,
+        // `recursive: false`) — exactly the #369 scenario. `config.roots`
+        // are always full-scrape targets regardless of `is_metadata_only`
+        // (see `#crawlUntilPendingClears`'s identical root-exclusion
+        // comment for why a root's row can be wrongly flagged by another
+        // page's anchor). `config.roots` is `ExURL#withoutHash` form (may
+        // keep basic-auth userinfo), but `pendingMetadataOnly` entries are
+        // always `url_refs.url` (`withoutHashAndAuth`, auth stripped —
+        // see `insert-page.ts`/`resolve-content-item-id.ts`) — re-parse
+        // each root through the same normalization before comparing, or a
+        // root URL carrying credentials would never match `rootKeys` and
+        // would wrongly stay in `metadataOnlyUrls` below (issue #369 code
+        // review).
+        const rootKeys = new Set(config.roots
+            .map((root) => parseUrl(root, config)?.withoutHashAndAuth)
+            .filter((root) => root !== undefined));
+        const metadataOnlyUrls = pendingMetadataOnly.filter((u) => !rootKeys.has(u));
         setupProgress?.onPhase?.(PHASE_LOADING_RESOURCES);
         const resources = await archive.getResourceUrlList(setupProgress?.onChunkProgress);
         setupProgress?.onPhase?.(PHASE_LOADING_SCRAPED_COUNT);
         const pagesScrapedOffset = await archive.getScrapedHtmlPageCount();
         setupProgress?.onPhase?.(PHASE_RESTORING_CRAWL_STATE);
-        orchestrator.#crawler.resume(pending, scraped, resources, pagesScrapedOffset);
+        orchestrator.#crawler.resume(pending, scraped, resources, pagesScrapedOffset, metadataOnlyUrls);
         if (initializedCallback) {
             await initializedCallback(orchestrator, config);
         }
@@ -1409,10 +2265,33 @@ export class CrawlerOrchestrator extends EventEmitter {
         log('URL %s', url.href);
         log('Config %O', config);
         await CrawlerOrchestrator.#preloadDnsBurnedHostCache(archive);
-        await orchestrator.crawling([url]);
+        await orchestrator.#crawlUntilPendingClears([url]);
         CrawlerOrchestrator.#finalizeCrawlSession(orchestrator);
         return orchestrator;
     }
+    /**
+     * Shared `PendingUrlsRemainError` recovery step for `append` and
+     * `retryFailed`'s catch blocks (issue #350 code review — `inventory` /
+     * `recrawl` reach the same outcome through their own `ingestionComplete`
+     * branch instead, which has no `.bak` left to clean up by the time it
+     * runs, so this helper is specific to the two `.bak`-restore-by-default
+     * catch shapes).
+     *
+     * `#crawlUntilPendingClears` has already released the archive handle
+     * and left the stub intact for `--resume`/`--retry-failed` by the time
+     * this runs; `write()` never ran, so the original archive file was
+     * never touched. Restoring `.bak` over it would be a wasted
+     * full-archive copy (and show a misleading "Restoring from backup"
+     * phase label) — only the now-unnecessary `.bak` needs cleaning up.
+     * @param setupProgress - Forwarded so the recovery phase label still
+     *   reaches the caller's setup `TaskList`.
+     * @param backupPath - The `.bak` path to delete (`unlinkFile`, ENOENT
+     *   ignored).
+     */
+    static async #abandonBackupOnPendingRemains(setupProgress, backupPath) {
+        setupProgress?.onPhase?.(RECOVERY_LEAVE_STATE_FOR_RESUME);
+        await ignoreEnoent(unlinkFile(backupPath));
+    }
     /**
      * Seeds {@link dnsBurnedHostCache} from `crawl_errors` history at re-open
      * (append / inventory / retryFailed / resume). Called after Archive.open
@@ -1435,8 +2314,40 @@ export class CrawlerOrchestrator extends EventEmitter {
         }
     }
     /**
-     * Persist one `inventory_runs` audit row inside the ingestion phase of a
-     * `--inventory` invocation, before the `.bak` is released. Lives as a
+     * Reads back this archive's prior `DedupeCapTracker` observations (see
+     * `Archive.listDedupeCapObservations`) and reconstructs them via
+     * `buildDedupeCapObservation`, for the same five resuming-session static
+     * methods that already call `archive.listDedupeCapShapeKeys()` to seed
+     * `CrawlConfig.preloadedStickyShapeKeys`. Unlike that sticky-shape
+     * preload, this read is skipped entirely when `dedupeCap` is `null` — a
+     * full per-page table scan is not worth paying on every resuming session
+     * that has `--dedupe-cap` disabled, whereas the shape-key `DISTINCT`
+     * query stays cheap enough to always run.
+     * @param archive - The opened archive whose scraped pages are read.
+     * @param dedupeCap - The resolved `CrawlConfig.dedupeCap` for this
+     *   session. `null` short-circuits to `[]` without touching the archive.
+     * @param onProgress - Forwarded to `archive.listDedupeCapObservations` —
+     *   see that method's docs.
+     * @returns Every qualifying page's reconstructed observation (rows with
+     *   no usable shape/meta signal are silently dropped — see
+     *   `buildDedupeCapObservation`).
+     */
+    static async #preloadDedupeCapObservations(archive, dedupeCap, onProgress) {
+        if (dedupeCap === null)
+            return [];
+        const rows = await archive.listDedupeCapObservations(onProgress);
+        const observations = [];
+        for (const row of rows) {
+            const observation = buildDedupeCapObservation(row);
+            if (observation) {
+                observations.push(observation);
+            }
+        }
+        return observations;
+    }
+    /**
+     * Persist one `list_reconcile_runs` audit row inside the ingestion phase
+     * of a `--inventory` invocation, before the `.bak` is released. Lives as a
      * static helper because the audit-row shape (timestamp stamping + label
      * auto-gen + the privacy-driven path elision documented below) is a
      * cohesive concern that benefits from staying outside the long
@@ -1451,7 +2362,7 @@ export class CrawlerOrchestrator extends EventEmitter {
      * `computeFileSha256` against the bytes it read from the input txt,
      * before the orchestrator was even invoked). The orchestrator boundary
      * deliberately never sees the absolute path — see
-     * {@link InventoryRunAggregates} for the privacy rationale.
+     * {@link ListReconcileRunAggregates} for the privacy rationale.
      *
      * **Audit-write failures abort the ingestion phase.** Swallowing them
      * would only be justified if the audit were the last write after the
@@ -1459,20 +2370,20 @@ export class CrawlerOrchestrator extends EventEmitter {
      * `.bak`-protected ingestion phase the
      * trade-off flips. A failed audit row is restorable: the outer catch
      * copies `.bak` back over the archive and the operator reruns the
-     * (short) ingestion from scratch. That keeps `inventory_runs` honest
-     * (no "ran but unrecorded" rows) at the cost of one rerun.
+     * (short) ingestion from scratch. That keeps `list_reconcile_runs`
+     * honest (no "ran but unrecorded" rows) at the cost of one rerun.
      *
      * Forward-compat: if an explicit `--label` flag is ever added, thread
      * `labelOverride` through {@link inventory} into the `aggregates`
      * shape so the auto-name can be overridden.
      * @param archive - The opened archive to write the audit row into.
-     * @param aggregates - The counts captured during the inventory pass; see {@link InventoryRunAggregates}.
+     * @param aggregates - The counts captured during the inventory pass; see {@link ListReconcileRunAggregates}.
      */
-    static async #writeInventoryRunRow(archive, aggregates) {
+    static async #writeListReconcileRunRow(archive, aggregates) {
         const ranAt = new Date().toISOString();
-        await archive.recordInventoryRun({
+        await archive.recordListReconcileRun({
             ran_at: ranAt,
-            list_label: `inventory-${ranAt}`,
+            list_label: `${aggregates.listLabelPrefix ?? 'inventory'}-${ranAt}`,
             source_file_sha256: aggregates.sourceFileSha256,
             total_lines: aggregates.inventoryUrlsCount,
             new_pages: aggregates.htmlSeedsCount,
@@ -1480,6 +2391,7 @@ export class CrawlerOrchestrator extends EventEmitter {
             scope_skipped: aggregates.outOfScope,
             exclude_skipped: aggregates.excludeSkipped,
             invalid_skipped: aggregates.invalidSkipped,
+            notes: aggregates.notes ?? null,
         });
     }
     /**
@@ -1514,6 +2426,37 @@ export class CrawlerOrchestrator extends EventEmitter {
         clearDnsBurnedHostCache();
     }
 }
+/**
+ * Resolves a `CrawlConfig.cwd` value to an absolute path before it is
+ * stamped as `Config.createdCwd` (issue #350). `cwd` is trusted as an
+ * absolute base everywhere else in this file (`path.resolve(cwd, ...)` for
+ * `absFilePath`/tmpDir), so a caller-supplied relative `cwd` already
+ * resolves against `process.cwd()` implicitly for archive placement — this
+ * makes that same resolution explicit for the value `Archive.resume` will
+ * later read back, so a relative `createdCwd` can never silently
+ * reintroduce the cwd-dependent resume path this column exists to fix.
+ * `path.resolve` is a no-op when `cwd` is already absolute.
+ * @param cwd - The `CrawlConfig.cwd` value (defaults to `process.cwd()` at each call site).
+ * @returns An absolute path.
+ */
+function resolveAbsoluteCwd(cwd) {
+    return path.resolve(process.cwd(), cwd);
+}
+/**
+ * Builds the `Config` patch that stamps `createdCwd` (issue #350) — the one
+ * field every stub-creating static factory (`crawling`/`append`/`inventory`/
+ * `recrawl`/`retryFailed`; `resume` deliberately excluded, it only reads
+ * this value) must set. Centralised so a future stub-creating mode spreads
+ * this into its `setConfig`/`updateConfig` call instead of hand-rolling
+ * `{ createdCwd: resolveAbsoluteCwd(cwd) }` and risking a forgotten
+ * `resolveAbsoluteCwd` wrap (see that function's JSDoc for why the
+ * resolution itself matters).
+ * @param cwd - The `CrawlConfig.cwd` value for this session.
+ * @returns A one-field `Partial<Config>` patch.
+ */
+function buildCreatedCwdPatch(cwd) {
+    return { createdCwd: resolveAbsoluteCwd(cwd) };
+}
 /**
  * Await a filesystem promise but silently swallow only `ENOENT` errors. Any
  * other failure (permissions, disk full, etc.) propagates so the caller can