@nitpicker/crawler 0.21.0 → 0.22.0

package/lib/crawler-orchestrator.d.ts

@@ -1,7 +1,9 @@
 import type { Config } from './archive/types.js';
+import type { DedupeCapObservation } from './crawler/dedupe/types.js';
 import type { NetworkProbe } from './crawler/probe-network.js';
-import type { InventoryMode } from './crawler/types.js';
+import type { CrawlRuntimeOptions, CrawlRuntimeOptionsPatch, InventoryMode } from './crawler/types.js';
 import type { CrawlEvent, SetupProgressCallbacks } from './types.js';
+import type { Lanes } from '@d-zero/dealer';
 import type { ExURL } from '@d-zero/shared/parse-url';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
 import Archive from './archive/archive.js';
@@ -37,6 +39,17 @@ interface CrawlConfig extends Config {
     list: boolean;
     /** Maximum number of retry attempts per URL on scrape failure. */
     retry: number;
+    /**
+     * Maximum number of whole-session auto-retry attempts (issue #350) when
+     * a crawl session ends with pages still pending — each attempt re-queues
+     * the current pending set and re-runs the crawl loop, with an
+     * exponential backoff between attempts (see
+     * `computeAutoRetryBackoffDelayMs`). `0` disables auto-retry entirely:
+     * any pending pages after the session's first (and only) crawl pass
+     * abort the session immediately. See
+     * `CrawlerOrchestrator`'s `#crawlUntilPendingClears` for the full loop.
+     */
+    maxAutoRetry: number;
     /** Whether to enable verbose logging output. */
     verbose: boolean;
     /** Custom User-Agent string for HTTP requests. */
@@ -75,12 +88,29 @@ interface CrawlConfig extends Config {
     dedupeMapCap: number;
     /**
      * See {@link CrawlerOptions.preloadedStickyShapeKeys}. Set internally by
-     * the four resuming-session static methods
-     * (`append`/`inventory`/`retryFailed`/`resume`), each independently
-     * calling `archive.listDedupeCapShapeKeys()`; not part of the public
-     * options a caller of those methods passes directly.
+     * the five resuming-session static methods
+     * (`append`/`inventory`/`recrawl`/`retryFailed`/`resume`), each
+     * independently calling `archive.listDedupeCapShapeKeys()`; not part of
+     * the public options a caller of those methods passes directly.
      */
     preloadedStickyShapeKeys: readonly string[];
+    /**
+     * See {@link CrawlerOptions.preloadedDedupeObservations}. Set internally
+     * by the same five resuming-session static methods as
+     * {@link preloadedStickyShapeKeys}, each calling
+     * `archive.listDedupeCapObservations()` and mapping the rows through
+     * `buildDedupeCapObservation`; not part of the public options a caller
+     * of those methods passes directly.
+     */
+    preloadedDedupeObservations: readonly DedupeCapObservation[];
+    /**
+     * A `Lanes` instance owned by the caller (typically the CLI), forwarded
+     * to {@link Crawler} so `deal()` reuses it instead of creating its own —
+     * see {@link CrawlerOptions.lanes}. Not part of {@link Config}: it is a
+     * live object, never persisted to (or read back from) the archive's
+     * `info` row.
+     */
+    lanes?: Lanes;
 }
 /**
  * Callback invoked after the CrawlerOrchestrator instance is fully initialized
@@ -100,7 +130,7 @@ interface InventorySource {
     sha256: string;
     /** The exact bytes of the source list file, archived verbatim. */
     bytes: Buffer;
-    /** Number of source-file lines the CLI warned-and-dropped for failing URL validation, before `inventoryUrls` was ever built. Recorded on the audit row as `inventory_runs.invalid_skipped`. */
+    /** Number of source-file lines the CLI warned-and-dropped for failing URL validation, before `inventoryUrls` was ever built. Recorded on the audit row as `list_reconcile_runs.invalid_skipped`. */
     invalidLineCount: number;
 }
 /**
@@ -159,16 +189,43 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * Sets up event listeners on the crawler, starts crawling, and resolves
      * when the crawl completes. Discovered pages, external pages, skipped pages,
      * and resources are forwarded to the archive for storage.
+     *
+     * Safe to call more than once on the same instance —
+     * `#crawlUntilPendingClears` (issue #350) re-invokes this for each
+     * auto-retry attempt against the same long-lived `#crawler`. `Crawler`
+     * (`TypedAwaitEventEmitter`) has no listener-removal API, so a second
+     * call cannot replace the first call's listeners — it can only stack
+     * another set alongside them. Every listener this method attaches is
+     * instead guarded by a monotonic generation counter (`isCurrent()`,
+     * defined below): once a later call bumps it, every earlier call's
+     * listeners permanently fail the check and become inert no-ops, leaving
+     * exactly the latest call's listeners actually writing anything.
      * @param list - The list of parsed URLs to crawl. May be empty when a resumed
      *   session already has pending pages queued (for example `--retry-failed`).
      * @param opts - Optional crawl overrides.
      * @param opts.recursive - Whether discovered URLs are followed. Defaults to
      *   `!fromList` (recursive unless the archive was created from a URL list), so
      *   existing callers keep their behaviour; the retry flow passes it explicitly.
+     * @param opts.suppressFlushNotice - Skip emitting `flushingPendingWrites`
+     *   (issue #350). Set by `#crawlUntilPendingClears` for every auto-retry
+     *   attempt after the first: that event starts the CLI's crawl-tail
+     *   `TaskList` (`attach-crawl-display.ts`), which must stay closed until
+     *   the whole retry loop is done — a second `deal()`/`Lanes` cycle
+     *   starting while that `TaskList` is still open would corrupt the
+     *   display (see ARCHITECTURE.md's `Lanes`/`Display` single-instance
+     *   invariant). The write-queue drain itself is unaffected; only the
+     *   CLI-facing progress event is skipped.
+     * @param opts.isRetryContinuation - Forwarded to `Crawler#start()`
+     *   (issue #350). Set by `#crawlUntilPendingClears` for every auto-retry
+     *   attempt after the first, so `#runDeal` preserves cross-attempt
+     *   learned state (known-good hosts, outage-detector window) instead of
+     *   discarding it as if this were an unrelated fresh session.
      * @returns A promise that resolves when crawling is complete.
      */
     crawling(list: ExURL[], opts?: {
         recursive?: boolean;
+        suppressFlushNotice?: boolean;
+        isRetryContinuation?: boolean;
     }): Promise<void>;
     /**
      * Kill any zombie Chromium processes that were not properly cleaned up.
@@ -184,6 +241,34 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * @returns An array of process IDs that should be terminated.
      */
     getUndeadPid(): never[];
+    /**
+     * Applies a runtime change to the in-progress crawl's tunable options
+     * (`parallels`/`interval`/the three exclude arrays — see
+     * {@link Crawler.updateRuntimeOptions}) and persists the resulting
+     * values to the archive's `info` row so a later `--resume` / `--append`
+     * / `--retry-failed` reuses them (`Archive#updateConfig`).
+     *
+     * The in-memory change takes effect immediately, before this method
+     * returns. The archive write is enqueued on {@link #writeQueue} — so it
+     * is ordered relative to the crawl's own page/resource writes — but not
+     * awaited: this method is synchronous so a caller (e.g. the CLI reading
+     * a console command) can report the new value back without waiting on
+     * disk I/O. A failure in that write does NOT abort the crawl or set
+     * {@link #archiveFailure} — see {@link #persistingRuntimeOptionsPatch}'s
+     * JSDoc for why this one write is deliberately exempted from the fatal
+     * handling every other archive write gets; it is only debug-logged
+     * (`DEBUG=Nitpicker:*`).
+     * @param patch - The runtime change to apply.
+     * @returns A snapshot of the tunable options after applying `patch`.
+     * @throws {RangeError} If `parallels` is present and not an integer `>= 1`, or `interval` is present and not an integer `>= 0`.
+     * @throws {TypeError} If any exclude entry is present and not a non-empty string.
+     * @example
+     * ```ts
+     * const snapshot = orchestrator.updateRuntimeOptions({ parallels: 4 });
+     * console.log(snapshot.parallels); // 4
+     * ```
+     */
+    updateRuntimeOptions(patch: CrawlRuntimeOptionsPatch): CrawlRuntimeOptions;
     /**
      * Write the archive to its configured file path.
      *
@@ -215,6 +300,7 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * @param initializedCallback - Optional callback invoked after initialization but before crawling starts.
      * @returns A promise that resolves to the CrawlerOrchestrator instance after crawling completes.
      * @throws {Error} If the URL list is empty or contains no valid URLs.
+     * @throws {PendingUrlsRemainError} When the crawl session ends with pages still pending after exhausting auto-retry.
      */
     static crawling(url: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback): Promise<CrawlerOrchestrator>;
     /**
@@ -226,7 +312,11 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * the expanded scope are demoted back to "needs scraping" so the next pass
      * re-fetches them as full internal pages. A `<archive>.bak` is created
      * before the crawl and removed on success; if the crawl throws, the backup
-     * is restored to keep the original archive intact.
+     * is restored to keep the original archive intact — except when the crawl
+     * ends with {@link PendingUrlsRemainError} (issue #350), where the
+     * un-packaged stub itself is the recovery path and the backup is instead
+     * left untouched (deleted, not restored — see
+     * {@link CrawlerOrchestrator.#abandonBackupOnPendingRemains}).
      *
      * List-mode archives (`info.fromList === true`) are rejected because their
      * pages are all metadata-only and cannot host a recursive append.
@@ -240,6 +330,7 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      *   this can't go through the orchestrator's event emitter (issue #294).
      * @returns The orchestrator instance after the append crawl completes.
      * @throws {Error} When `newUrls` is empty, the archive is in list mode, or it cannot be parsed.
+     * @throws {PendingUrlsRemainError} When the crawl session ends with pages still pending after exhausting auto-retry.
      */
     static append(archivePath: string, newUrls: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback, setupProgress?: SetupProgressCallbacks): Promise<CrawlerOrchestrator>;
     /**
@@ -326,8 +417,75 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      *   #294).
      * @returns The orchestrator instance after a successful inventory pass.
      * @throws {Error} When `inventoryUrls` is empty or the archive is in list mode. Unresolved pending URLs from a previous crawl do NOT throw — see step 3.
+     * @throws {PendingUrlsRemainError} When the crawl session ends with pages still pending after exhausting auto-retry.
      */
     static inventory(archivePath: string, inventoryUrls: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback, source?: InventorySource | null, setupProgress?: SetupProgressCallbacks): Promise<CrawlerOrchestrator>;
+    /**
+     * Re-fetch pages named by an operator-supplied URL list, importing any
+     * URL the archive does not yet track as a new inventory seed.
+     *
+     * `recrawl` is `retryFailed`'s un-scrape combined with `inventory`'s
+     * novel-URL ingestion, run inside one `.bak`-protected window: URLs in
+     * `recrawlUrls` that already exist as `content_items` rows are reset back
+     * to pending via {@link Archive.resetPagesByUrls} (see that method for the
+     * conservative exclusion rules — redirect sources, intentionally-skipped
+     * pages, and external pages are matched but never reset), while URLs the
+     * archive has never seen are ingested exactly as `inventory` does (see
+     * {@link CrawlerOrchestrator.inventory}'s JSDoc for that half's contract).
+     * Existing *resources* matched by the list are neither resettable nor
+     * novel — `resource_items` is first-write-wins (a known deviation, see
+     * ARCHITECTURE.md), so a resource re-fetch would not update anything; the
+     * function reports how many list entries fell into this bucket via
+     * `setupProgress.onLog` without acting on them.
+     *
+     * Unlike `inventory`, whose sole early-return condition is "no novel
+     * URLs", `recrawl` also has existing pages to act on — the `.bak` is
+     * skipped only when BOTH `existingPageUrls` (reset candidates) AND
+     * `novelUrls` (ingestion candidates) are empty.
+     *
+     * **Strict-pending gap**: `getCrawlingState()`'s pending set only includes
+     * a `scraped = 0` row that is either anchor-referenced or explicitly
+     * labelled (see that function's JSDoc). When `recrawlUrls` contains pages
+     * that link to each other, resetting one page also deletes its outgoing
+     * `anchor_edges` — so a `source = 'crawled'` sibling that was reset in the
+     * same pass can lose its only anchor referrer and fall out of the strict
+     * pending set, silently skipping its re-fetch. `retryFailed` never hits
+     * this because a failed page's referrers are not themselves reset. The
+     * fix: every URL `Archive.resetPagesByUrls` actually reset is merged into
+     * the pending list handed to `Crawler#resume` regardless of what the
+     * strict scan finds, deduplicated by `LinkList.add`'s `protocolAgnosticKey`
+     * check. A Ctrl+C between the reset and the scrape phase loses this
+     * synthetic merge (it lives only in memory) — `crawl --resume` recovers
+     * whatever the strict-pending scan finds on its own, and re-running
+     * `--recrawl` with the same list recovers the rest, matching the
+     * "un-picked seeds" recovery contract `getCrawlingState`'s JSDoc already
+     * documents for `inventory`.
+     *
+     * **Stale analyze findings**: resetting a page deletes its
+     * `analysis_violations` rows (see {@link resetPagesByUrls}'s JSDoc) so a
+     * re-fetched page never shows findings from HTML that no longer exists,
+     * but other `analyze` outputs (e.g. Discrepancies plugin reports) are not
+     * page-scoped and cannot be selectively invalidated. When at least one
+     * page was reset, a `crawlSessionNotice` is emitted after the crawl
+     * completes recommending `analyze` be re-run before the next `report`.
+     * @param archivePath - Absolute or relative path to the existing `.nitpicker`.
+     * @param recrawlUrls - URLs to match against the archive (existing pages
+     *   are reset; unknown URLs are ingested as new inventory seeds).
+     * @param options - Optional config overrides applied on top of the archived config.
+     * @param initializedCallback - Optional callback invoked after initialization but before crawling resumes.
+     * @param source - The CLI's already-read URL list source bytes, archived
+     *   for audit purposes — see {@link InventorySource}. `null` for
+     *   programmatic callers with no source file.
+     * @param setupProgress - Optional progress callbacks for the setup phase
+     *   (untar, `.bak` copy, URL classification, reset, seed ingestion, state
+     *   rebuild) that runs before `initializedCallback` — see
+     *   {@link SetupProgressCallbacks} for why this can't go through the
+     *   orchestrator's event emitter (issue #294).
+     * @returns The orchestrator instance after the recrawl completes.
+     * @throws {Error} When `recrawlUrls` is empty or the archive is in list mode.
+     * @throws {PendingUrlsRemainError} When the crawl session ends with pages still pending after exhausting auto-retry.
+     */
+    static recrawl(archivePath: string, recrawlUrls: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback, source?: InventorySource | null, setupProgress?: SetupProgressCallbacks): Promise<CrawlerOrchestrator>;
     /**
      * Re-fetch previously-failed pages in an existing `.nitpicker` archive.
      *
@@ -351,7 +509,10 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      *
      * A `<archive>.bak` is created before any DB mutation and removed on success;
      * if the crawl throws, the backup is restored to keep the original archive
-     * intact.
+     * intact — except when the crawl ends with {@link PendingUrlsRemainError}
+     * (issue #350), where the un-packaged stub itself is the recovery path and
+     * the backup is instead left untouched (deleted, not restored — see
+     * {@link CrawlerOrchestrator.#abandonBackupOnPendingRemains}).
      *
      * List-mode archives (`info.fromList === true`) are rejected for the same
      * reason as {@link CrawlerOrchestrator.append}: their pages are metadata-only.
@@ -364,6 +525,7 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      *   this can't go through the orchestrator's event emitter (issue #294).
      * @returns The orchestrator instance after the retry crawl completes.
      * @throws {Error} When the archive is in list mode or has no parseable roots.
+     * @throws {PendingUrlsRemainError} When the crawl session ends with pages still pending after exhausting auto-retry.
      */
     static retryFailed(archivePath: string, options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback, setupProgress?: SetupProgressCallbacks): Promise<CrawlerOrchestrator>;
     /**
@@ -385,6 +547,7 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      *   tmpDir IS the source of truth).
      * @returns A promise that resolves to the CrawlerOrchestrator instance after crawling completes.
      * @throws {Error} If the archived URL is invalid.
+     * @throws {PendingUrlsRemainError} When the crawl session ends with pages still pending after exhausting auto-retry.
      */
     static resume(stubPath: string, options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback, setupProgress?: SetupProgressCallbacks): Promise<CrawlerOrchestrator>;
 }