@nitpicker/crawler 0.21.0 → 0.22.0

package/lib/crawler.d.ts

@@ -40,7 +40,10 @@ export { RESUME_SETUP_PHASES } from './resume-setup-phases.js';
 export { APPEND_SETUP_PHASES } from './append-setup-phases.js';
 export { RETRY_FAILED_SETUP_PHASES } from './retry-failed-setup-phases.js';
 export { INVENTORY_SETUP_PHASES } from './inventory-setup-phases.js';
+export { RECRAWL_SETUP_PHASES } from './recrawl-setup-phases.js';
 export { SETUP_RECOVERY_PHASE_LABELS } from './setup-recovery-phase-labels.js';
+export { PendingUrlsRemainError } from './pending-urls-remain-error.js';
+export { computeAutoRetryBackoffDelayMs } from './compute-auto-retry-backoff-delay.js';
 export * from './types.js';
 export * from './crawler/types.js';
 export { classifyErrorKind } from './classify-error-kind.js';

package/lib/crawler.js

@@ -39,7 +39,10 @@ export { RESUME_SETUP_PHASES } from './resume-setup-phases.js';
 export { APPEND_SETUP_PHASES } from './append-setup-phases.js';
 export { RETRY_FAILED_SETUP_PHASES } from './retry-failed-setup-phases.js';
 export { INVENTORY_SETUP_PHASES } from './inventory-setup-phases.js';
+export { RECRAWL_SETUP_PHASES } from './recrawl-setup-phases.js';
 export { SETUP_RECOVERY_PHASE_LABELS } from './setup-recovery-phase-labels.js';
+export { PendingUrlsRemainError } from './pending-urls-remain-error.js';
+export { computeAutoRetryBackoffDelayMs } from './compute-auto-retry-backoff-delay.js';
 export * from './types.js';
 export * from './crawler/types.js';
 export { classifyErrorKind } from './classify-error-kind.js';

package/lib/delay-or-abort.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Waits `ms` milliseconds, or returns immediately if `signal` aborts first
+ * (issue #350's auto-retry backoff — up to 5 minutes, which a library
+ * consumer calling `CrawlerOrchestrator#abort()` must be able to cut short
+ * immediately rather than waiting out).
+ *
+ * WHY NOT `Promise.race([delay(ms), abortPromise])`: `@d-zero/shared`'s
+ * `delay()` takes no `AbortSignal` and cannot cancel its own internal
+ * `setTimeout` — racing it leaves that timer running (and its executor
+ * closure alive) for the full `ms` even after the abort side already won,
+ * which is exactly the "loser-side timer never cleared" pattern
+ * ARCHITECTURE.md's invariants call out (`raceWithTimeout`'s own docs).
+ * This instead owns a single `setTimeout` directly and clears it — via the
+ * abort listener when the signal fires first, or by letting it fire
+ * naturally and detaching the listener — on whichever path resolves.
+ * @param ms - Milliseconds to wait.
+ * @param signal - Aborting this resolves the returned promise immediately.
+ * @returns A promise that resolves after `ms`, or immediately if already aborted.
+ */
+export declare function delayOrAbort(ms: number, signal: AbortSignal): Promise<void>;

package/lib/delay-or-abort.js

@@ -0,0 +1,35 @@
+/**
+ * Waits `ms` milliseconds, or returns immediately if `signal` aborts first
+ * (issue #350's auto-retry backoff — up to 5 minutes, which a library
+ * consumer calling `CrawlerOrchestrator#abort()` must be able to cut short
+ * immediately rather than waiting out).
+ *
+ * WHY NOT `Promise.race([delay(ms), abortPromise])`: `@d-zero/shared`'s
+ * `delay()` takes no `AbortSignal` and cannot cancel its own internal
+ * `setTimeout` — racing it leaves that timer running (and its executor
+ * closure alive) for the full `ms` even after the abort side already won,
+ * which is exactly the "loser-side timer never cleared" pattern
+ * ARCHITECTURE.md's invariants call out (`raceWithTimeout`'s own docs).
+ * This instead owns a single `setTimeout` directly and clears it — via the
+ * abort listener when the signal fires first, or by letting it fire
+ * naturally and detaching the listener — on whichever path resolves.
+ * @param ms - Milliseconds to wait.
+ * @param signal - Aborting this resolves the returned promise immediately.
+ * @returns A promise that resolves after `ms`, or immediately if already aborted.
+ */
+export function delayOrAbort(ms, signal) {
+    if (signal.aborted) {
+        return Promise.resolve();
+    }
+    return new Promise((resolve) => {
+        const timer = setTimeout(() => {
+            signal.removeEventListener('abort', onAbort);
+            resolve();
+        }, ms);
+        const onAbort = () => {
+            clearTimeout(timer);
+            resolve();
+        };
+        signal.addEventListener('abort', onAbort, { once: true });
+    });
+}

package/lib/pending-urls-remain-error.d.ts

@@ -0,0 +1,35 @@
+import type { PendingUrlsRemainReason } from './types.js';
+/**
+ * Thrown by `CrawlerOrchestrator`'s auto-retry loop when a crawl session
+ * ends with `content_items` rows still unscraped and no further retry is
+ * warranted (issue #350). The archive is deliberately left un-packaged —
+ * see the loop's own JSDoc for the `.nitpicker` ⟹ pending = 0 invariant this
+ * enforces — so the message points the operator at the stub left behind.
+ */
+export declare class PendingUrlsRemainError extends Error {
+    /** Attempts actually run before giving up (0 when `--max-auto-retry 0`). */
+    readonly attemptsMade: number;
+    /** The configured `--max-auto-retry` ceiling. */
+    readonly maxAutoRetry: number;
+    /** Pending URL count at the moment retrying was abandoned. */
+    readonly pendingCount: number;
+    /** Why the loop stopped. */
+    readonly reason: PendingUrlsRemainReason;
+    /** Absolute path of the stub (tmpDir) left behind for `--resume`/`--retry-failed`. */
+    readonly stubPath: string;
+    /**
+     * @param params - See the matching readonly property for each field's meaning.
+     * @param params.pendingCount
+     * @param params.attemptsMade
+     * @param params.maxAutoRetry
+     * @param params.reason
+     * @param params.stubPath
+     */
+    constructor(params: {
+        pendingCount: number;
+        attemptsMade: number;
+        maxAutoRetry: number;
+        reason: PendingUrlsRemainReason;
+        stubPath: string;
+    });
+}

package/lib/pending-urls-remain-error.js

@@ -0,0 +1,43 @@
+/**
+ * Thrown by `CrawlerOrchestrator`'s auto-retry loop when a crawl session
+ * ends with `content_items` rows still unscraped and no further retry is
+ * warranted (issue #350). The archive is deliberately left un-packaged —
+ * see the loop's own JSDoc for the `.nitpicker` ⟹ pending = 0 invariant this
+ * enforces — so the message points the operator at the stub left behind.
+ */
+export class PendingUrlsRemainError extends Error {
+    /** Attempts actually run before giving up (0 when `--max-auto-retry 0`). */
+    attemptsMade;
+    /** The configured `--max-auto-retry` ceiling. */
+    maxAutoRetry;
+    /** Pending URL count at the moment retrying was abandoned. */
+    pendingCount;
+    /** Why the loop stopped. */
+    reason;
+    /** Absolute path of the stub (tmpDir) left behind for `--resume`/`--retry-failed`. */
+    stubPath;
+    /**
+     * @param params - See the matching readonly property for each field's meaning.
+     * @param params.pendingCount
+     * @param params.attemptsMade
+     * @param params.maxAutoRetry
+     * @param params.reason
+     * @param params.stubPath
+     */
+    constructor(params) {
+        const { pendingCount, attemptsMade, maxAutoRetry, reason, stubPath } = params;
+        const reasonText = reason === 'no-progress'
+            ? `an auto-retry attempt made no progress (pending count did not decrease)`
+            : `all ${maxAutoRetry} auto-retry attempt(s) were exhausted`;
+        super(`${pendingCount} page(s) remain pending after ${attemptsMade} auto-retry attempt(s) — ${reasonText}. ` +
+            `The archive was left un-packaged at: ${stubPath}. ` +
+            `Run \`crawl ${stubPath} --resume\` (or \`--retry-failed\` once packaged) to continue, ` +
+            'or rerun with a higher --max-auto-retry.');
+        this.name = 'PendingUrlsRemainError';
+        this.pendingCount = pendingCount;
+        this.attemptsMade = attemptsMade;
+        this.maxAutoRetry = maxAutoRetry;
+        this.reason = reason;
+        this.stubPath = stubPath;
+    }
+}

package/lib/recrawl-setup-phases.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Ordered `SetupProgressCallbacks.onPhase` labels
+ * `CrawlerOrchestrator.recrawl` announces, in call order, before its
+ * `initializedCallback` fires. Single source of truth for both the `onPhase`
+ * call sites in `crawler-orchestrator.ts` and the CLI's setup task list
+ * (`create-setup-task-list.ts`), which pre-builds one task-list row per
+ * entry — duplicating this sequence by hand at the CLI layer would silently
+ * drift the moment a phase is added, renamed, or reordered here.
+ *
+ * Identical to `INVENTORY_SETUP_PHASES` except for one extra step —
+ * `'Resetting matched pages'`, right after `'Backing up archive'` — since
+ * `recrawl` un-scrapes URLs that already exist as pages in addition to
+ * importing the URLs `inventory` handles (novel URLs the archive does not
+ * yet track). This is the **superset** sequence — `recrawl` has three
+ * `initializedCallback` call sites, each reached after a different prefix of
+ * this list depending on the input:
+ * - no matched pages AND no novel URLs (everything already known / out of
+ *   scope): stops after index 3 (`'Checking for already-known URLs'`), skips
+ *   the `.bak` backup entirely
+ * - novel URLs are all non-HTML and nothing was reset: stops after index 8
+ *   (`'Recording excluded pages'`) — nothing to render, no crawl state to
+ *   rebuild
+ * - at least one reset page or novel HTML seed: runs the full 13-phase
+ *   sequence
+ *
+ * The CLI's setup task list pre-builds all 13 rows and marks whichever
+ * suffix never got announced as skipped once `initializedCallback` fires —
+ * it does not need to know in advance which of the three prefixes a given
+ * run will take. Note `'Loading crawl state'` appears twice (indices 2 and
+ * 9): the first read is scoped to the pre-ingestion pending-URL warning, the
+ * second re-reads state after the reset pages and new inventory seeds have
+ * been written.
+ *
+ * On failure, a `SETUP_RECOVERY_PHASE_LABELS` entry
+ * (`'Restoring archive from backup'` or `'Persisting ingested inventory
+ * state'`) is announced instead of continuing this sequence.
+ */
+export declare const RECRAWL_SETUP_PHASES: readonly ["Extracting archive", "Loading archive config", "Loading crawl state", "Checking for already-known URLs", "Backing up archive", "Resetting matched pages", "Recording non-HTML resources", "Recording HTML seed pages", "Recording excluded pages", "Loading crawl state", "Loading resource list", "Loading scraped page count", "Restoring crawl state"];

package/lib/recrawl-setup-phases.js

@@ -0,0 +1,52 @@
+/**
+ * Ordered `SetupProgressCallbacks.onPhase` labels
+ * `CrawlerOrchestrator.recrawl` announces, in call order, before its
+ * `initializedCallback` fires. Single source of truth for both the `onPhase`
+ * call sites in `crawler-orchestrator.ts` and the CLI's setup task list
+ * (`create-setup-task-list.ts`), which pre-builds one task-list row per
+ * entry — duplicating this sequence by hand at the CLI layer would silently
+ * drift the moment a phase is added, renamed, or reordered here.
+ *
+ * Identical to `INVENTORY_SETUP_PHASES` except for one extra step —
+ * `'Resetting matched pages'`, right after `'Backing up archive'` — since
+ * `recrawl` un-scrapes URLs that already exist as pages in addition to
+ * importing the URLs `inventory` handles (novel URLs the archive does not
+ * yet track). This is the **superset** sequence — `recrawl` has three
+ * `initializedCallback` call sites, each reached after a different prefix of
+ * this list depending on the input:
+ * - no matched pages AND no novel URLs (everything already known / out of
+ *   scope): stops after index 3 (`'Checking for already-known URLs'`), skips
+ *   the `.bak` backup entirely
+ * - novel URLs are all non-HTML and nothing was reset: stops after index 8
+ *   (`'Recording excluded pages'`) — nothing to render, no crawl state to
+ *   rebuild
+ * - at least one reset page or novel HTML seed: runs the full 13-phase
+ *   sequence
+ *
+ * The CLI's setup task list pre-builds all 13 rows and marks whichever
+ * suffix never got announced as skipped once `initializedCallback` fires —
+ * it does not need to know in advance which of the three prefixes a given
+ * run will take. Note `'Loading crawl state'` appears twice (indices 2 and
+ * 9): the first read is scoped to the pre-ingestion pending-URL warning, the
+ * second re-reads state after the reset pages and new inventory seeds have
+ * been written.
+ *
+ * On failure, a `SETUP_RECOVERY_PHASE_LABELS` entry
+ * (`'Restoring archive from backup'` or `'Persisting ingested inventory
+ * state'`) is announced instead of continuing this sequence.
+ */
+export const RECRAWL_SETUP_PHASES = [
+    'Extracting archive',
+    'Loading archive config',
+    'Loading crawl state',
+    'Checking for already-known URLs',
+    'Backing up archive',
+    'Resetting matched pages',
+    'Recording non-HTML resources',
+    'Recording HTML seed pages',
+    'Recording excluded pages',
+    'Loading crawl state',
+    'Loading resource list',
+    'Loading scraped page count',
+    'Restoring crawl state',
+];

package/lib/resume-setup-phases.d.ts

@@ -13,4 +13,4 @@
  * recovery phase and no branching: every `resume` call announces exactly
  * these seven phases in this order.
  */
-export declare const RESUME_SETUP_PHASES: readonly ["Reconnecting to archive", "Loading archive config", "Loading dedupe-cap shape keys", "Loading crawl state", "Loading resource list", "Loading scraped page count", "Restoring crawl state"];
+export declare const RESUME_SETUP_PHASES: readonly ["Reconnecting to archive", "Loading archive config", "Loading dedupe-cap state", "Loading crawl state", "Loading resource list", "Loading scraped page count", "Restoring crawl state"];

package/lib/resume-setup-phases.js

@@ -16,7 +16,11 @@
 export const RESUME_SETUP_PHASES = [
     'Reconnecting to archive',
     'Loading archive config',
-    'Loading dedupe-cap shape keys',
+    // Renamed from 'Loading dedupe-cap shape keys' — this phase now covers
+    // both the confirmed-capped shape-key read AND the not-yet-capped shape
+    // observation replay (`archive.listDedupeCapObservations` /
+    // `#preloadDedupeCapObservations`), not just the former.
+    'Loading dedupe-cap state',
     'Loading crawl state',
     'Loading resource list',
     'Loading scraped page count',

package/lib/retry-failed-setup-phases.d.ts

@@ -16,4 +16,4 @@
  * (`'Restoring archive from backup'`) is announced instead of continuing
  * this sequence.
  */
-export declare const RETRY_FAILED_SETUP_PHASES: readonly ["Extracting archive", "Loading archive config", "Backing up archive", "Resetting failed pages", "Loading dedupe-cap shape keys", "Loading crawl state", "Loading resource list", "Loading scraped page count", "Restoring crawl state"];
+export declare const RETRY_FAILED_SETUP_PHASES: readonly ["Extracting archive", "Loading archive config", "Backing up archive", "Resetting failed pages", "Loading dedupe-cap state", "Loading crawl state", "Loading resource list", "Loading scraped page count", "Restoring crawl state"];

package/lib/setup-recovery-phase-labels.d.ts

@@ -10,12 +10,21 @@
  * stays visible, rather than trying to guess in advance where in the
  * success-path sequence a failure might interrupt it.
  *
- * - `'Restoring archive from backup'` — `append` / `inventory` /
- *   `retryFailed` all take this path when anything from the post-backup
- *   setup step through the crawl itself throws.
- * - `'Persisting ingested inventory state'` — `inventory`-only: once its
- *   pre-insert + audit-row write has committed, a later failure (state
- *   rebuild or the crawl itself) must persist the ingested rows to disk
- *   instead of restoring the pre-inventory `.bak`, or they would be lost.
+ * - `'Restoring archive from backup'` — `append` / `inventory` / `recrawl` /
+ *   `retryFailed` take this path when a failure that is NOT
+ *   `PendingUrlsRemainError` occurs from the post-backup setup step through
+ *   the crawl itself.
+ * - `'Leaving crawl state for --resume'` — all four of the above instead
+ *   take this path once the failure is specifically `PendingUrlsRemainError`
+ *   (issue #350's auto-retry loop giving up with pages still pending): the
+ *   `.bak` is deleted (nothing to restore — `write()` never ran, so the
+ *   original `.nitpicker` was never touched) and the stub tmpDir is left
+ *   intact instead, so the operator recovers via `crawl --resume` /
+ *   `--retry-failed`. `inventory` / `recrawl` additionally reach this same
+ *   label for any OTHER scrape-phase failure once their pre-insert + audit
+ *   row has committed (`ingestionComplete`) — restoring the pre-inventory
+ *   `.bak` at that point would silently wipe the already-committed seeds.
+ *   The archive is deliberately NOT packaged in either case (see
+ *   `CrawlerOrchestrator`'s `.nitpicker` ⟹ pending = 0 invariant).
  */
-export declare const SETUP_RECOVERY_PHASE_LABELS: readonly ["Restoring archive from backup", "Persisting ingested inventory state"];
+export declare const SETUP_RECOVERY_PHASE_LABELS: readonly ["Restoring archive from backup", "Leaving crawl state for --resume"];

package/lib/setup-recovery-phase-labels.js

@@ -10,15 +10,24 @@
  * stays visible, rather than trying to guess in advance where in the
  * success-path sequence a failure might interrupt it.
  *
- * - `'Restoring archive from backup'` — `append` / `inventory` /
- *   `retryFailed` all take this path when anything from the post-backup
- *   setup step through the crawl itself throws.
- * - `'Persisting ingested inventory state'` — `inventory`-only: once its
- *   pre-insert + audit-row write has committed, a later failure (state
- *   rebuild or the crawl itself) must persist the ingested rows to disk
- *   instead of restoring the pre-inventory `.bak`, or they would be lost.
+ * - `'Restoring archive from backup'` — `append` / `inventory` / `recrawl` /
+ *   `retryFailed` take this path when a failure that is NOT
+ *   `PendingUrlsRemainError` occurs from the post-backup setup step through
+ *   the crawl itself.
+ * - `'Leaving crawl state for --resume'` — all four of the above instead
+ *   take this path once the failure is specifically `PendingUrlsRemainError`
+ *   (issue #350's auto-retry loop giving up with pages still pending): the
+ *   `.bak` is deleted (nothing to restore — `write()` never ran, so the
+ *   original `.nitpicker` was never touched) and the stub tmpDir is left
+ *   intact instead, so the operator recovers via `crawl --resume` /
+ *   `--retry-failed`. `inventory` / `recrawl` additionally reach this same
+ *   label for any OTHER scrape-phase failure once their pre-insert + audit
+ *   row has committed (`ingestionComplete`) — restoring the pre-inventory
+ *   `.bak` at that point would silently wipe the already-committed seeds.
+ *   The archive is deliberately NOT packaged in either case (see
+ *   `CrawlerOrchestrator`'s `.nitpicker` ⟹ pending = 0 invariant).
  */
 export const SETUP_RECOVERY_PHASE_LABELS = [
     'Restoring archive from backup',
-    'Persisting ingested inventory state',
+    'Leaving crawl state for --resume',
 ];

package/lib/types.d.ts

@@ -1,17 +1,19 @@
 import type { APPEND_SETUP_PHASES } from './append-setup-phases.js';
 import type { INVENTORY_SETUP_PHASES } from './inventory-setup-phases.js';
+import type { RECRAWL_SETUP_PHASES } from './recrawl-setup-phases.js';
 import type { RESUME_SETUP_PHASES } from './resume-setup-phases.js';
 import type { RETRY_FAILED_SETUP_PHASES } from './retry-failed-setup-phases.js';
 import type { SETUP_RECOVERY_PHASE_LABELS } from './setup-recovery-phase-labels.js';
 import type { CrawlerError, PageData } from './utils/types/types.js';
 /**
  * Every label `SetupProgressCallbacks.onPhase` can be called with, across
- * all four `CrawlerOrchestrator` setup sequences plus the failure-only
+ * all five `CrawlerOrchestrator` setup sequences plus the failure-only
  * recovery phases. See `RESUME_SETUP_PHASES` / `APPEND_SETUP_PHASES` /
  * `RETRY_FAILED_SETUP_PHASES` / `INVENTORY_SETUP_PHASES` /
- * `SETUP_RECOVERY_PHASE_LABELS` for what each label means and when it fires.
+ * `RECRAWL_SETUP_PHASES` / `SETUP_RECOVERY_PHASE_LABELS` for what each label
+ * means and when it fires.
  */
-export type SetupPhaseLabel = (typeof RESUME_SETUP_PHASES)[number] | (typeof APPEND_SETUP_PHASES)[number] | (typeof RETRY_FAILED_SETUP_PHASES)[number] | (typeof INVENTORY_SETUP_PHASES)[number] | (typeof SETUP_RECOVERY_PHASE_LABELS)[number];
+export type SetupPhaseLabel = (typeof RESUME_SETUP_PHASES)[number] | (typeof APPEND_SETUP_PHASES)[number] | (typeof RETRY_FAILED_SETUP_PHASES)[number] | (typeof INVENTORY_SETUP_PHASES)[number] | (typeof RECRAWL_SETUP_PHASES)[number] | (typeof SETUP_RECOVERY_PHASE_LABELS)[number];
 /**
  * Progress callbacks for the setup phase of `CrawlerOrchestrator.append` /
  * `inventory` / `retryFailed` / `resume` (issue #294) — everything from
@@ -71,17 +73,17 @@ export interface SetupProgressCallbacks {
     onLog?: (message: string) => void;
 }
 /**
- * Aggregate counts captured during a `--inventory` invocation, forwarded to
- * `#writeInventoryRunRow` so the audit log row is consistent between the
- * HTML-seed branch and the non-HTML-only branch of
- * `CrawlerOrchestrator.inventory`.
+ * Aggregate counts captured during a `--inventory` or `--recrawl`
+ * invocation, forwarded to `#writeListReconcileRunRow` so the audit log row
+ * is consistent across every branch of `CrawlerOrchestrator.inventory` and
+ * `CrawlerOrchestrator.recrawl` that reaches ingestion.
  *
  * Spelled out here (not inlined at the call site) so a new field added to
  * the audit row has a single edit point and so each field's semantics are
- * documented per-property rather than scattered across the two emit sites.
+ * documented per-property rather than scattered across the emit sites.
  */
-export interface InventoryRunAggregates {
-    /** `inventoryUrls.length` as received by `CrawlerOrchestrator.inventory` — the CLI (`inventoryCrawl`) has already warned-and-dropped unparseable-URL lines before this point, so this counts valid URLs, not raw source-file lines. Stored verbatim as `inventory_runs.total_lines`. */
+export interface ListReconcileRunAggregates {
+    /** `inventoryUrls.length` as received by `CrawlerOrchestrator.inventory` — the CLI (`inventoryCrawl`) has already warned-and-dropped unparseable-URL lines before this point, so this counts valid URLs, not raw source-file lines. Stored verbatim as `list_reconcile_runs.total_lines`. */
     inventoryUrlsCount: number;
     /** Number of novel URLs classified as HTML and queued for render. Stored as `new_pages` (excludes anchor-discovered descendants — those add later via the crawler graph and are NOT counted here). */
     htmlSeedsCount: number;
@@ -94,7 +96,7 @@ export interface InventoryRunAggregates {
     /**
      * SHA-256 hex digest of the source `.txt`, **pre-computed by the caller**
      * (typically the CLI's `inventoryCrawl`). Stored verbatim as
-     * `inventory_runs.source_file_sha256`.
+     * `list_reconcile_runs.source_file_sha256`.
      *
      * Pre-computation lifts the absolute path off the orchestrator
      * boundary entirely — the path is privacy-sensitive (leaks
@@ -108,11 +110,28 @@ export interface InventoryRunAggregates {
     /**
      * Number of source-file lines the CLI warned-and-dropped for failing
      * URL validation, before `inventoryUrlsCount` was ever counted. Stored
-     * verbatim as `inventory_runs.invalid_skipped`. `null` for programmatic
+     * verbatim as `list_reconcile_runs.invalid_skipped`. `null` for programmatic
      * callers that built `inventoryUrls` in-memory — there is no source
      * file, so no line was ever dropped as invalid.
      */
     invalidSkipped: number | null;
+    /**
+     * Overrides the audit row's `list_label` prefix (before the `-${ranAt}`
+     * timestamp suffix `#writeListReconcileRunRow` always appends). Omit for
+     * the default `'inventory'`; `CrawlerOrchestrator.recrawl` passes
+     * `'recrawl'` so the two invocation kinds stay distinguishable in
+     * `query list-reconcile-runs` output despite sharing one audit table.
+     */
+    listLabelPrefix?: string;
+    /**
+     * Free-form text stored verbatim as `list_reconcile_runs.notes`. `recrawl`
+     * uses this to record how many existing pages it reset back to
+     * pending — a fact `--inventory` never produces and that therefore has
+     * no dedicated column (adding one would leave it `NULL` on every
+     * `--inventory` row forever). `null`/omitted leaves the column `NULL`,
+     * matching `--inventory`'s existing rows.
+     */
+    notes?: string | null;
 }
 /**
  * Coarse cause of a crawl/scrape failure.
@@ -165,6 +184,18 @@ export interface InventoryRunAggregates {
  * PUPPETEER_FALLBACK entry).
  */
 export type ErrorKind = 'dns' | 'dns-transient' | 'connection-refused' | 'connection-reset' | 'connection-timeout' | 'tls' | 'local-network' | 'parse-error' | 'client-blocked' | 'redirect-loop' | 'timeout' | 'protocol' | 'unknown';
+/**
+ * Why an auto-retry loop stopped without reaching `pending === 0` (issue
+ * #350):
+ *
+ * - `'exhausted'` — every configured `--max-auto-retry` attempt ran and
+ *   pending URLs still remain.
+ * - `'no-progress'` — a single attempt made no dent in the pending count
+ *   (it stayed the same or grew), so the remaining budget is abandoned
+ *   early instead of burning every attempt against a cause retrying will
+ *   not fix (e.g. a wholesale host outage).
+ */
+export type PendingUrlsRemainReason = 'exhausted' | 'no-progress';
 /**
  * Event map for the `CrawlerOrchestrator` class.
  *
@@ -260,6 +291,25 @@ export interface CrawlEvent {
         /** The formatted, ready-to-display notice text. */
         message: string;
     };
+    /**
+     * Emitted once before each auto-retry attempt's backoff wait begins
+     * (issue #350's `#crawlUntilPendingClears`) — see that method's JSDoc for
+     * the loop it belongs to. Purely observational: the CLI does not
+     * subscribe to this for display (the orchestrator prints its own
+     * `console.error` line for the same reason `networkOutageConfirmed`
+     * does), but tests and other programmatic consumers can use it to
+     * observe retry progress without capturing stdout.
+     */
+    autoRetryWaiting: {
+        /** The 1-indexed attempt about to run after this wait. */
+        attempt: number;
+        /** The configured `--max-auto-retry` ceiling. */
+        maxAttempts: number;
+        /** Pending URL count that triggered this retry. */
+        pendingCount: number;
+        /** Backoff delay before the attempt starts, in milliseconds. */
+        delayMs: number;
+    };
     /**
      * Emitted when an error occurs during crawling or archiving.
      */

package/lib/utils/compute-file-sha256.d.ts

@@ -2,7 +2,7 @@
  * Compute the SHA-256 digest of an in-memory byte buffer.
  *
  * Used by `CrawlerOrchestrator.inventory` to fingerprint the source URL
- * list — the digest is both recorded on the `inventory_runs` audit row
+ * list — the digest is both recorded on the `list_reconcile_runs` audit row
  * (the content-identity key for "was this exact list applied before") and
  * used as the file name under which the raw list is archived
  * (`Archive.saveInventorySourceList`).

package/lib/utils/compute-file-sha256.js

@@ -3,7 +3,7 @@ import { createHash } from 'node:crypto';
  * Compute the SHA-256 digest of an in-memory byte buffer.
  *
  * Used by `CrawlerOrchestrator.inventory` to fingerprint the source URL
- * list — the digest is both recorded on the `inventory_runs` audit row
+ * list — the digest is both recorded on the `list_reconcile_runs` audit row
  * (the content-identity key for "was this exact list applied before") and
  * used as the file name under which the raw list is archived
  * (`Archive.saveInventorySourceList`).

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@nitpicker/crawler",
-	"version": "0.21.0",
+	"version": "0.22.0",
 	"description": "Web crawler engine with headless browser rendering and archive storage",
 	"author": "D-ZERO",
 	"license": "Apache-2.0",
@@ -30,8 +30,8 @@
 		"clean": "tsc --build --clean"
 	},
 	"dependencies": {
-		"@d-zero/beholder": "4.2.3",
-		"@d-zero/dealer": "1.13.1",
+		"@d-zero/beholder": "5.0.0",
+		"@d-zero/dealer": "1.14.0",
 		"@d-zero/fs": "0.2.7",
 		"@d-zero/shared": "0.23.0",
 		"ansi-colors": "4.1.3",
@@ -51,5 +51,5 @@
 		"@types/tar": "7.0.87",
 		"@types/unzipper": "0.10.11"
 	},
-	"gitHead": "233d081009aeeeb2d2622536ca29b22693ba65ca"
+	"gitHead": "934b1714f4394effb3865d84bf8348ed74f45840"
 }

package/lib/archive/db-ops/inventory/record-inventory-run.d.ts

@@ -1,21 +0,0 @@
-import type { InventoryRunMeta } from '../../types.js';
-import type { Knex } from 'knex';
-/**
- * Appends one row to the `inventory_runs` audit log.
- *
- * Called by `CrawlerOrchestrator.inventory` on every successful
- * `--inventory <list>` invocation so the archive carries a durable
- * record of which deploy list was applied when and at what scale —
- * the operational question "did we apply last month's list" the
- * archive itself can answer without consulting external bookkeeping.
- *
- * Append-only. There is intentionally no UPDATE path and no UNIQUE
- * constraint on `source_file_sha256`; two applies of the same list
- * each get their own row. Duplicate detection is left to readers —
- * the hash is recorded as the content-identity key they would use.
- * Field-level NULL semantics live on {@link InventoryRunMeta}.
- * @param knex - Knex query builder connected to the archive DB.
- * @param meta - The run metadata to record. Only `ran_at` is required.
- * @returns The autoincremented `id` of the newly-inserted row.
- */
-export declare function recordInventoryRun(knex: Knex, meta: InventoryRunMeta): Promise<number>;

package/lib/archive/migrate-inventory-runs-invalid-skipped.d.ts

@@ -1,16 +0,0 @@
-import type { Knex } from 'knex';
-/**
- * Adds the `inventory_runs.invalid_skipped` column to archives created
- * before it existed. `CREATE TABLE IF NOT EXISTS` (used for `inventory_runs`
- * itself) cannot retrofit a new column onto an already-existing table, so
- * this lightweight, `hasColumn`-guarded `ALTER TABLE` runs on every
- * `initSchema` call — idempotent, and self-healing for archives whose
- * provisioning crashed partway through.
- * @param instance - The Knex query builder instance connected to the database.
- * @param onLog - Called instead of `console.error` when this migration
- *   actually applies (issue #294: a bare `console.error` here can fire
- *   while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
- *   `Archive.open`, corrupting its cursor tracking). Falls back to
- *   `console.error` when omitted (direct/test callers).
- */
-export declare function migrateInventoryRunsInvalidSkipped(instance: Knex, onLog?: (message: string) => void): Promise<void>;

package/lib/archive/migrate-inventory-runs-invalid-skipped.js

@@ -1,35 +0,0 @@
-/**
- * Adds the `inventory_runs.invalid_skipped` column to archives created
- * before it existed. `CREATE TABLE IF NOT EXISTS` (used for `inventory_runs`
- * itself) cannot retrofit a new column onto an already-existing table, so
- * this lightweight, `hasColumn`-guarded `ALTER TABLE` runs on every
- * `initSchema` call — idempotent, and self-healing for archives whose
- * provisioning crashed partway through.
- * @param instance - The Knex query builder instance connected to the database.
- * @param onLog - Called instead of `console.error` when this migration
- *   actually applies (issue #294: a bare `console.error` here can fire
- *   while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
- *   `Archive.open`, corrupting its cursor tracking). Falls back to
- *   `console.error` when omitted (direct/test callers).
- */
-export async function migrateInventoryRunsInvalidSkipped(instance, onLog) {
-    const hasTable = await instance.schema.hasTable('inventory_runs');
-    if (!hasTable) {
-        return;
-    }
-    const hasColumn = await instance.schema.hasColumn('inventory_runs', 'invalid_skipped');
-    if (hasColumn) {
-        return;
-    }
-    await instance.schema.table('inventory_runs', (t) => {
-        t.integer('invalid_skipped');
-    });
-    const message = '[migrate] inventory_runs.invalid_skipped column added';
-    if (onLog) {
-        onLog(message);
-    }
-    else {
-        // eslint-disable-next-line no-console
-        console.error(message);
-    }
-}