@nitpicker/crawler 0.18.1 → 0.19.0

package/lib/crawler-orchestrator.js

@@ -1,10 +1,12 @@
-import { copyFile, unlink as unlinkFile } from 'node:fs/promises';
+import { unlink as unlinkFile } from 'node:fs/promises';
 import path from 'node:path';
 import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
 import { sortUrl } from '@d-zero/shared/sort-url';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
 import pkg from '../package.json' with { type: 'json' };
+import { APPEND_SETUP_PHASES } from './append-setup-phases.js';
 import Archive from './archive/archive.js';
+import { copyFileWithProgress } from './archive/filesystem/copy-file-with-progress.js';
 import { REQUIRED_FORMAT_VERSION } from './archive/meta/assert-compatible-version.js';
 import { clearDestinationCache } from './crawler/clear-destination-cache.js';
 import { clearDnsBurnedHostCache } from './crawler/clear-dns-burned-host-cache.js';
@@ -18,11 +20,16 @@ import { PreloadShortCircuitError } from './crawler/preload-short-circuit-error.
 import { protocolAgnosticKey } from './crawler/protocol-agnostic-key.js';
 import { shouldSkipUrl } from './crawler/should-skip-url.js';
 import { crawlerLog, log } from './debug.js';
+import { INVENTORY_SETUP_PHASES } from './inventory-setup-phases.js';
 import { normalizeToArray } from './normalize-to-array.js';
 import { resolveOutputPath } from './resolve-output-path.js';
 import { resourceRowToLookupResult } from './resource-row-to-lookup-result.js';
+import { RESUME_SETUP_PHASES } from './resume-setup-phases.js';
+import { RETRY_FAILED_SETUP_PHASES } from './retry-failed-setup-phases.js';
+import { SETUP_RECOVERY_PHASE_LABELS } from './setup-recovery-phase-labels.js';
 import { cleanObject } from './utils/object/clean-object.js';
 import { WriteQueue } from './write-queue.js';
+const [RECOVERY_RESTORE_FROM_BACKUP, RECOVERY_PERSIST_INGESTED_STATE] = SETUP_RECOVERY_PHASE_LABELS;
 /**
  * Default list of external URL prefixes excluded from crawling.
  * Includes social media sharing endpoints that are commonly linked
@@ -53,10 +60,13 @@ export const DEFAULT_EXCLUDED_EXTERNAL_URLS = [
  * writes the final archive file. It emits events defined by {@link CrawlEvent}.
  *
  * Instances are created via the static factory methods {@link CrawlerOrchestrator.crawling}
- * or {@link CrawlerOrchestrator.resume}; the constructor is private.
+ * or {@link CrawlerOrchestrator.resume}; the constructor is private. Implements
+ * `Symbol.asyncDispose` so callers can use `await using` to close the archive
+ * and reap zombie Chromium processes on scope exit instead of a manual
+ * `try`/`finally` around `archive.close()` + `garbageCollect()`.
  * @example
  * ```ts
- * const orchestrator = await CrawlerOrchestrator.crawling(['https://example.com'], { recursive: true });
+ * await using orchestrator = await CrawlerOrchestrator.crawling(['https://example.com'], { recursive: true });
  * await orchestrator.write();
  * ```
  */
@@ -177,6 +187,36 @@ export class CrawlerOrchestrator extends EventEmitter {
             preloadedStickyShapeKeys: options?.preloadedStickyShapeKeys ?? [],
         });
     }
+    /**
+     * Enables `await using orchestrator = ...`. Closes the archive (write
+     * or remove tmpDir + release the lock, per {@link Archive.close}) and
+     * then reaps any zombie Chromium processes via {@link garbageCollect} —
+     * the same two-step teardown every CLI crawl command previously
+     * repeated by hand in a `finally` block.
+     *
+     * Relays `Archive.close()`'s recovery-write progress (issue #294) as
+     * `recoveringArchiveWrite`/`writeStep`/`writeTarProgress` — the same
+     * events `write()` emits — for the rare case where the file doesn't
+     * exist on disk yet at dispose time (e.g. an explicit `write()` call
+     * threw partway through). A CLI listener whose display is still open at
+     * that point (it hadn't yet seen `writeFileEnd`) picks these up for
+     * free; one that already tore down after the earlier failure silently
+     * drops them, same as any other post-close display update.
+     */
+    async [Symbol.asyncDispose]() {
+        await this.#archive.close({
+            onRecoveryStart: () => {
+                void this.emit('recoveringArchiveWrite', {});
+            },
+            onStep: (step) => {
+                void this.emit('writeStep', { step });
+            },
+            onTarProgress: (writtenBytes, totalBytes) => {
+                void this.emit('writeTarProgress', { writtenBytes, totalBytes });
+            },
+        });
+        this.garbageCollect();
+    }
     /**
      * Abort the current crawl operation.
      *
@@ -228,9 +268,9 @@ export class CrawlerOrchestrator extends EventEmitter {
                     .catch((writeError) => reject(writeError));
                 void this.emit('error', error);
             });
-            this.#crawler.on('page', ({ result, source }) => {
+            this.#crawler.on('page', ({ result, source, bodyHash }) => {
                 writeQueue
-                    .enqueue(() => this.#archive.setPage(result, source))
+                    .enqueue(() => this.#archive.setPage(result, source, bodyHash))
                     .catch((error) => reject(error));
             });
             this.#crawler.on('externalPage', ({ result, source }) => {
@@ -256,8 +296,13 @@ export class CrawlerOrchestrator extends EventEmitter {
             });
             this.#crawler.on('networkOutageConfirmed', ({ startedAt, detectedAt, probeHost, triggerErrorCount, triggerHostCount }) => {
                 crawlerLog('Network outage confirmed: probeHost=%s triggerErrorCount=%d triggerHostCount=%d', probeHost, triggerErrorCount, triggerHostCount);
-                // event notice; mirrors `#finalizeCrawlSession`'s unconditional
-                // `console.error` for the DNS-burn short-circuit summary.
+                // Rare anomaly notice, printed unconditionally even though this
+                // fires while `deal()`'s own crawl-time `Lanes` is actively
+                // rendering (issue #294: unlike the crawl-tail notices reported
+                // via the `crawlSessionNotice` event, this one has no
+                // listener-based route available mid-crawl — visibility during
+                // the outage takes priority over the display glitch this
+                // causes).
                 console.error(`[network] outage suspected — pausing workers (probe host: ${probeHost ?? 'none'})`);
                 writeQueue
                     .enqueue(async () => {
@@ -348,6 +393,13 @@ export class CrawlerOrchestrator extends EventEmitter {
                     .catch((error) => reject(error));
             });
             this.#crawler.on('crawlEnd', () => {
+                // Read BEFORE enqueuing the dedupeCap-finalize closure below
+                // (issue #294) so this reflects genuine backlog from the
+                // crawl's own page/resource writes, not the finalize
+                // closure's own, always-present entry.
+                if (writeQueue.pending > 0) {
+                    void this.emit('flushingPendingWrites', { pending: writeQueue.pending });
+                }
                 // Deferred to INSIDE a queued closure, not read synchronously
                 // here, for the same reason `networkOutageRecovered`'s handler
                 // defers reading `#openNetworkOutageId`: a `dedupeCap` event's
@@ -439,14 +491,38 @@ export class CrawlerOrchestrator extends EventEmitter {
      * the time `write()` is called those tables are already populated.
      * This method just tars.
      *
-     * Emits `writeFileStart` before writing and `writeFileEnd` after
-     * the write completes successfully.
+     * Emits `writeFileStart` before writing and `writeFileEnd` after the
+     * write completes successfully. Also relays `Archive.write()`'s
+     * per-step (`writeStep`) and tar-byte (`writeTarProgress`) progress
+     * (issue #294) — tarring a 15 GB+ archive can take minutes, and without
+     * these events a CLI listener has no way to show it isn't hung.
      */
     async write() {
         void this.emit('writeFileStart', { filePath: this.#archive.filePath });
-        await this.#archive.write();
+        await this.#archive.write({
+            onStep: (step) => {
+                void this.emit('writeStep', { step });
+            },
+            onTarProgress: (writtenBytes, totalBytes) => {
+                void this.emit('writeTarProgress', { writtenBytes, totalBytes });
+            },
+        });
         void this.emit('writeFileEnd', { filePath: this.#archive.filePath });
     }
+    /**
+     * Assign natural URL sort order to every internal page, relaying chunk
+     * progress through the `sortingUrls` event (issue #294). Always runs
+     * after `crawling()` has returned, i.e. once `initializedCallback` has
+     * already had a chance to attach listeners — unlike the setup-phase
+     * work in `append`/`inventory`/`retryFailed`, this has an orchestrator
+     * instance to emit from, so it goes through the event emitter rather
+     * than a `SetupProgressCallbacks` callback.
+     */
+    async #setUrlOrder() {
+        await this.#archive.setUrlOrder((processed, total) => {
+            void this.emit('sortingUrls', { processed, total });
+        });
+    }
     /**
      * Create a new CrawlerOrchestrator instance and start crawling the given URLs.
      *
@@ -523,9 +599,9 @@ export class CrawlerOrchestrator extends EventEmitter {
         log('Config %O', config);
         await orchestrator.crawling(list);
         log('Crawling completed');
-        CrawlerOrchestrator.#finalizeCrawlSession();
+        CrawlerOrchestrator.#finalizeCrawlSession(orchestrator);
         log('Set order natural URL sort');
-        await archive.setUrlOrder();
+        await orchestrator.#setUrlOrder();
         log('Sorting done');
         return orchestrator;
     }
@@ -546,10 +622,15 @@ export class CrawlerOrchestrator extends EventEmitter {
      * @param newUrls - New root URLs to add and crawl.
      * @param options - Optional config overrides applied on top of the archived config.
      * @param initializedCallback - Optional callback invoked after initialization but before crawling resumes.
+     * @param setupProgress - Optional progress callbacks for the setup phase
+     *   (untar, `.bak` copy, repromote, state rebuild) that runs before
+     *   `initializedCallback` — see {@link SetupProgressCallbacks} for why
+     *   this can't go through the orchestrator's event emitter (issue #294).
      * @returns The orchestrator instance after the append crawl completes.
      * @throws {Error} When `newUrls` is empty, the archive is in list mode, or it cannot be parsed.
      */
-    static async append(archivePath, newUrls, options, initializedCallback) {
+    static async append(archivePath, newUrls, options, initializedCallback, setupProgress) {
+        const [PHASE_EXTRACTING, PHASE_LOADING_CONFIG, PHASE_BACKING_UP, PHASE_REPROMOTING, PHASE_LOADING_DEDUPE_KEYS, PHASE_LOADING_CRAWL_STATE, PHASE_LOADING_RESOURCES, PHASE_LOADING_SCRAPED_COUNT, PHASE_RESTORING_CRAWL_STATE,] = APPEND_SETUP_PHASES;
         if (newUrls.length === 0) {
             throw new Error('append: newUrls is empty');
         }
@@ -559,16 +640,20 @@ export class CrawlerOrchestrator extends EventEmitter {
             : path.resolve(cwd, archivePath);
         // See `ArchiveOpenOptions.openPluginData` for why this must be `true`
         // on every writer path that calls `write()`.
+        setupProgress?.onPhase?.(PHASE_EXTRACTING);
         const archive = await Archive.open({
             filePath: absFilePath,
             cwd,
             openPluginData: true,
+            onExtractProgress: setupProgress?.onExtractProgress,
+            onLog: setupProgress?.onLog,
         });
         // Any throw between here and the successful return must release the
         // archive lock and clean up tmpDir; the caller's `close()` only runs on
         // the happy path. Errors from `close()` itself are intentionally
         // best-effort: the original error is what matters.
         try {
+            setupProgress?.onPhase?.(PHASE_LOADING_CONFIG);
             const archived = await archive.getConfig();
             if (archived.fromList) {
                 throw new Error('Cannot append to a list-mode archive: this archive was created with --list/--list-file and contains metadata-only pages. Create a fresh archive instead.');
@@ -588,7 +673,8 @@ export class CrawlerOrchestrator extends EventEmitter {
                 baseUrl: mergedRoots[0],
             };
             const backupPath = absFilePath + '.bak';
-            await copyFile(absFilePath, backupPath);
+            setupProgress?.onPhase?.(PHASE_BACKING_UP);
+            await copyFileWithProgress(absFilePath, backupPath, setupProgress?.onCopyProgress);
             try {
                 await archive.updateConfig(mergedConfig);
                 const scopeMap = new Map();
@@ -599,19 +685,25 @@ export class CrawlerOrchestrator extends EventEmitter {
                     const existing = scopeMap.get(parsed.hostname) ?? [];
                     scopeMap.set(parsed.hostname, [...existing, parsed]);
                 }
-                await archive.repromoteExternalPages(scopeMap, archived);
+                setupProgress?.onPhase?.(PHASE_REPROMOTING);
+                await archive.repromoteExternalPages(scopeMap, archived, setupProgress?.onChunkProgress);
                 // Seed the sticky set from prior sessions' confirmed traps so
                 // `--append` does not pay the cost of re-discovering them (see
                 // `DedupeCapTracker`'s constructor JSDoc).
+                setupProgress?.onPhase?.(PHASE_LOADING_DEDUPE_KEYS);
                 const preloadedStickyShapeKeys = await archive.listDedupeCapShapeKeys();
                 const orchestrator = new CrawlerOrchestrator(archive, {
                     ...mergedConfig,
                     roots: mergedRoots,
                     preloadedStickyShapeKeys,
                 });
+                setupProgress?.onPhase?.(PHASE_LOADING_CRAWL_STATE);
                 const { scraped, pending } = await archive.getCrawlingState();
-                const resources = await archive.getResourceUrlList();
+                setupProgress?.onPhase?.(PHASE_LOADING_RESOURCES);
+                const resources = await archive.getResourceUrlList(setupProgress?.onChunkProgress);
+                setupProgress?.onPhase?.(PHASE_LOADING_SCRAPED_COUNT);
                 const pagesScrapedOffset = await archive.getScrapedHtmlPageCount();
+                setupProgress?.onPhase?.(PHASE_RESTORING_CRAWL_STATE);
                 orchestrator.#crawler.resume(pending, scraped, resources, pagesScrapedOffset);
                 if (initializedCallback) {
                     await initializedCallback(orchestrator, mergedConfig);
@@ -622,14 +714,15 @@ export class CrawlerOrchestrator extends EventEmitter {
                 log('Merged roots %O', mergedRoots);
                 await CrawlerOrchestrator.#preloadDnsBurnedHostCache(archive);
                 await orchestrator.crawling(newParsed);
-                CrawlerOrchestrator.#finalizeCrawlSession();
-                await archive.setUrlOrder();
+                CrawlerOrchestrator.#finalizeCrawlSession(orchestrator);
+                await orchestrator.#setUrlOrder();
                 await ignoreEnoent(unlinkFile(backupPath));
                 return orchestrator;
             }
             catch (error) {
                 try {
-                    await copyFile(backupPath, absFilePath);
+                    setupProgress?.onPhase?.(RECOVERY_RESTORE_FROM_BACKUP);
+                    await copyFileWithProgress(backupPath, absFilePath, setupProgress?.onCopyProgress);
                     await ignoreEnoent(unlinkFile(backupPath));
                 }
                 catch (restoreError) {
@@ -723,10 +816,16 @@ export class CrawlerOrchestrator extends EventEmitter {
      *   URLs. Pass `null` for programmatic callers that built
      *   `inventoryUrls` in-memory; the audit row's `source_file_sha256`
      *   column will be `NULL` and no source list is archived.
+     * @param setupProgress - Optional progress callbacks for the setup phase
+     *   (untar, scope classification, bulk inserts, state rebuild) that runs
+     *   before `initializedCallback` — see {@link SetupProgressCallbacks} for
+     *   why this can't go through the orchestrator's event emitter (issue
+     *   #294).
      * @returns The orchestrator instance after a successful inventory pass.
      * @throws {Error} When `inventoryUrls` is empty or the archive is in list mode. Unresolved pending URLs from a previous crawl do NOT throw — see step 3.
      */
-    static async inventory(archivePath, inventoryUrls, options, initializedCallback, source = null) {
+    static async inventory(archivePath, inventoryUrls, options, initializedCallback, source = null, setupProgress) {
+        const [PHASE_EXTRACTING, PHASE_LOADING_CONFIG, PHASE_LOADING_CRAWL_STATE_PRE, PHASE_CHECKING_KNOWN_URLS, PHASE_BACKING_UP, PHASE_RECORDING_NON_HTML, PHASE_RECORDING_HTML_SEEDS, PHASE_RECORDING_EXCLUDED, PHASE_LOADING_CRAWL_STATE_POST, PHASE_LOADING_RESOURCES, PHASE_LOADING_SCRAPED_COUNT, PHASE_RESTORING_CRAWL_STATE,] = INVENTORY_SETUP_PHASES;
         if (inventoryUrls.length === 0) {
             throw new Error('inventory: URL list is empty');
         }
@@ -736,16 +835,21 @@ export class CrawlerOrchestrator extends EventEmitter {
             : path.resolve(cwd, archivePath);
         // See `ArchiveOpenOptions.openPluginData` for why this must be `true`
         // on every writer path that calls `write()`.
+        setupProgress?.onPhase?.(PHASE_EXTRACTING);
         const archive = await Archive.open({
             filePath: absFilePath,
             cwd,
             openPluginData: true,
+            onExtractProgress: setupProgress?.onExtractProgress,
+            onLog: setupProgress?.onLog,
         });
         try {
+            setupProgress?.onPhase?.(PHASE_LOADING_CONFIG);
             const archived = await archive.getConfig();
             if (archived.fromList) {
                 throw new Error('Cannot run inventory on a list-mode archive: this archive was created with --list/--list-file and contains metadata-only pages. Create a fresh archive instead.');
             }
+            setupProgress?.onPhase?.(PHASE_LOADING_CRAWL_STATE_PRE);
             const { pending } = await archive.getCrawlingState();
             if (pending.length > 0) {
                 // `getCrawlingState` returns the STRICT pending set — in-scope,
@@ -759,8 +863,28 @@ export class CrawlerOrchestrator extends EventEmitter {
                 // enough — the inventory pass continues and the crawled-wins
                 // source priority keeps stale labels stable even if some of
                 // the strict-pending rows happen to land on inventory seeds.
-                // eslint-disable-next-line no-console -- operator-facing warning, must be visible regardless of DEBUG filters
-                console.warn(`inventory: archive has ${pending.length} pending URLs from a previous crawl. Proceeding — crawled-wins priority keeps their labels stable. Consider \`--resume\` first if you want the prior work finalized.`);
+                //
+                // Routed through `setupProgress.onLog` (issue #294 code
+                // review), not a bare `console.warn`: the `'Loading crawl
+                // state'` row is active at this exact point (`onPhase` just
+                // above), and a direct stream write here corrupts the setup
+                // `TaskList`'s cursor tracking the same way self-healing
+                // migration notices did. `onLog` sets this row's message,
+                // which then freezes as its permanent `done` text once the
+                // next `onPhase` call moves past it — still visible in the
+                // final terminal output, just not printed as its own
+                // interrupting line. Falls back to `console.warn` under
+                // `--silent` (`setupProgress` is `null` there, so there is no
+                // row to set this on, and the warning would otherwise never
+                // surface at all).
+                const message = `inventory: archive has ${pending.length} pending URLs from a previous crawl. Proceeding — crawled-wins priority keeps their labels stable. Consider \`--resume\` first if you want the prior work finalized.`;
+                if (setupProgress?.onLog) {
+                    setupProgress.onLog(message);
+                }
+                else {
+                    // eslint-disable-next-line no-console -- --silent has no TaskList row to report through
+                    console.warn(message);
+                }
             }
             // Archive the exact source bytes before scope classification, so
             // even a run that discards every URL (out-of-scope or already
@@ -800,6 +924,7 @@ export class CrawlerOrchestrator extends EventEmitter {
             // to mirror what `resolveContentItemId` / `insertResource` actually store.
             // Two independent reads — Promise.all halves the wait on large
             // archives where each `WHERE url IN (?)` chunk costs real I/O.
+            setupProgress?.onPhase?.(PHASE_CHECKING_KNOWN_URLS);
             const candidateUrls = inScope.map((u) => u.withoutHashAndAuth);
             const [existingPageUrlList, existingResourceUrlList] = await Promise.all([
                 archive.getExistingPageUrls(candidateUrls),
@@ -860,7 +985,8 @@ export class CrawlerOrchestrator extends EventEmitter {
                 return orchestrator;
             }
             const backupPath = absFilePath + '.bak';
-            await copyFile(absFilePath, backupPath);
+            setupProgress?.onPhase?.(PHASE_BACKING_UP);
+            await copyFileWithProgress(absFilePath, backupPath, setupProgress?.onCopyProgress);
             // Ingestion (pre-insert + audit) is `.bak`-protected — a failure
             // there restores the archive and the operator reruns. Once
             // ingestion completes and the `.bak` is released, the scrape
@@ -934,6 +1060,7 @@ export class CrawlerOrchestrator extends EventEmitter {
                 // inside the `.bak`-protected window on large inventory
                 // lists; the chunked bulk path collapses N round-trips
                 // to N/500.
+                setupProgress?.onPhase?.(PHASE_RECORDING_NON_HTML);
                 await archive.insertInventoryResources(nonHtmlSeeds);
                 // Pre-insert HTML seeds as `scraped = 0`,
                 // `source = 'inventory-seed'` placeholders *before* the
@@ -941,6 +1068,7 @@ export class CrawlerOrchestrator extends EventEmitter {
                 // cannot lose the URL. The strict-pending set picks
                 // these rows up on the next `--resume` via the
                 // `OR p.source != 'crawled'` clause.
+                setupProgress?.onPhase?.(PHASE_RECORDING_HTML_SEEDS);
                 await archive.insertInventorySeeds(htmlSeeds);
                 // Record exclude-matched novel URLs as terminal skipped pages
                 // (`is_skipped=1`, `skip_reason='excluded'`,
@@ -949,6 +1077,7 @@ export class CrawlerOrchestrator extends EventEmitter {
                 // excluded URLs, so the archive looks identical no matter
                 // how the URL was discovered. Inside the `.bak` window for
                 // the same all-or-nothing reason as the seed inserts above.
+                setupProgress?.onPhase?.(PHASE_RECORDING_EXCLUDED);
                 await archive.insertInventorySkippedPages(excludedNovelUrls);
                 log('[inventory] %d HTML seed(s), %d non-HTML resource(s), %d skipped page(s) recorded', htmlSeeds.length, nonHtmlSeeds.length, excludedNovelUrls.length);
                 // Audit row is written *inside* the `.bak` window: a libsql
@@ -1015,15 +1144,19 @@ export class CrawlerOrchestrator extends EventEmitter {
                     // `retryFailed` uses to drive the dealer from the
                     // pending set alone (see retryFailed's
                     // `crawling([], { recursive })` invocation).
+                    setupProgress?.onPhase?.(PHASE_LOADING_CRAWL_STATE_POST);
                     const { scraped: scrapedAfter, pending: pendingAfter } = await archive.getCrawlingState();
-                    const resources = await archive.getResourceUrlList();
+                    setupProgress?.onPhase?.(PHASE_LOADING_RESOURCES);
+                    const resources = await archive.getResourceUrlList(setupProgress?.onChunkProgress);
                     // Pre-existing rendered HTML page count seeds the
                     // session-spanning `pagesScraped` counter so the progress
                     // header reads `internalDone(cumulative pagesScraped)`
                     // rather than session-only — matches the `append` /
                     // `retryFailed` / `resume` paths and avoids users reading
                     // the parenthesised number as "inner pages dropped to N".
+                    setupProgress?.onPhase?.(PHASE_LOADING_SCRAPED_COUNT);
                     const pagesScrapedOffset = await archive.getScrapedHtmlPageCount();
+                    setupProgress?.onPhase?.(PHASE_RESTORING_CRAWL_STATE);
                     orchestrator.#crawler.resume(pendingAfter, scrapedAfter, resources, pagesScrapedOffset);
                     if (initializedCallback) {
                         await initializedCallback(orchestrator, baseConfig);
@@ -1033,8 +1166,8 @@ export class CrawlerOrchestrator extends EventEmitter {
                     log('HTML seeds %O', htmlSeeds.map((u) => u.href));
                     await CrawlerOrchestrator.#preloadDnsBurnedHostCache(archive);
                     await orchestrator.crawling([], { recursive: true });
-                    CrawlerOrchestrator.#finalizeCrawlSession();
-                    await archive.setUrlOrder();
+                    CrawlerOrchestrator.#finalizeCrawlSession(orchestrator);
+                    await orchestrator.#setUrlOrder();
                     return orchestrator;
                 }
                 // Only non-HTML URLs were imported — nothing left to render,
@@ -1043,7 +1176,7 @@ export class CrawlerOrchestrator extends EventEmitter {
                 if (initializedCallback) {
                     await initializedCallback(orchestrator, baseConfig);
                 }
-                await archive.setUrlOrder();
+                await orchestrator.#setUrlOrder();
                 return orchestrator;
             }
             catch (error) {
@@ -1064,6 +1197,7 @@ export class CrawlerOrchestrator extends EventEmitter {
                     // catch's `close()` becomes a no-op for the destructive
                     // step and only runs `releaseLock` cleanup.
                     try {
+                        setupProgress?.onPhase?.(RECOVERY_PERSIST_INGESTED_STATE);
                         await archive.write();
                         await archive.releaseHandle();
                     }
@@ -1073,7 +1207,8 @@ export class CrawlerOrchestrator extends EventEmitter {
                     throw error;
                 }
                 try {
-                    await copyFile(backupPath, absFilePath);
+                    setupProgress?.onPhase?.(RECOVERY_RESTORE_FROM_BACKUP);
+                    await copyFileWithProgress(backupPath, absFilePath, setupProgress?.onCopyProgress);
                     await ignoreEnoent(unlinkFile(backupPath));
                 }
                 catch (restoreError) {
@@ -1117,25 +1252,34 @@ export class CrawlerOrchestrator extends EventEmitter {
      * @param archivePath - Absolute or relative path to the existing `.nitpicker`.
      * @param options - Optional config overrides applied on top of the archived config.
      * @param initializedCallback - Optional callback invoked after initialization but before crawling resumes.
+     * @param setupProgress - Optional progress callbacks for the setup phase
+     *   (untar, `.bak` copy, reset, state rebuild) that runs before
+     *   `initializedCallback` — see {@link SetupProgressCallbacks} for why
+     *   this can't go through the orchestrator's event emitter (issue #294).
      * @returns The orchestrator instance after the retry crawl completes.
      * @throws {Error} When the archive is in list mode or has no parseable roots.
      */
-    static async retryFailed(archivePath, options, initializedCallback) {
+    static async retryFailed(archivePath, options, initializedCallback, setupProgress) {
+        const [PHASE_EXTRACTING, PHASE_LOADING_CONFIG, PHASE_BACKING_UP, PHASE_RESETTING_FAILED, PHASE_LOADING_DEDUPE_KEYS, PHASE_LOADING_CRAWL_STATE, PHASE_LOADING_RESOURCES, PHASE_LOADING_SCRAPED_COUNT, PHASE_RESTORING_CRAWL_STATE,] = RETRY_FAILED_SETUP_PHASES;
         const cwd = options?.cwd ?? process.cwd();
         const absFilePath = path.isAbsolute(archivePath)
             ? archivePath
             : path.resolve(cwd, archivePath);
         // See `ArchiveOpenOptions.openPluginData` for why this must be `true`
         // on every writer path that calls `write()`.
+        setupProgress?.onPhase?.(PHASE_EXTRACTING);
         const archive = await Archive.open({
             filePath: absFilePath,
             cwd,
             openPluginData: true,
+            onExtractProgress: setupProgress?.onExtractProgress,
+            onLog: setupProgress?.onLog,
         });
         // Any throw between here and the successful return must release the
         // archive lock and clean up tmpDir; the caller's `close()` only runs on
         // the happy path.
         try {
+            setupProgress?.onPhase?.(PHASE_LOADING_CONFIG);
             const archived = await archive.getConfig();
             if (archived.fromList) {
                 throw new Error('Cannot retry a list-mode archive: this archive was created with --list/--list-file and contains metadata-only pages. Create a fresh archive instead.');
@@ -1152,37 +1296,45 @@ export class CrawlerOrchestrator extends EventEmitter {
                 baseUrl: archived.baseUrl,
             };
             const backupPath = absFilePath + '.bak';
-            await copyFile(absFilePath, backupPath);
+            setupProgress?.onPhase?.(PHASE_BACKING_UP);
+            await copyFileWithProgress(absFilePath, backupPath, setupProgress?.onCopyProgress);
             try {
-                const resetUrls = await archive.resetFailedPages();
+                setupProgress?.onPhase?.(PHASE_RESETTING_FAILED);
+                const resetUrls = await archive.resetFailedPages(setupProgress?.onChunkProgress);
                 log('Start retrying failed pages');
                 log('Archive %s', absFilePath);
                 log('Reset %d failed page(s)', resetUrls.length);
                 // Seed the sticky set from prior sessions' confirmed traps so
                 // `--retry-failed` does not pay the cost of re-discovering
                 // them (see `DedupeCapTracker`'s constructor JSDoc).
+                setupProgress?.onPhase?.(PHASE_LOADING_DEDUPE_KEYS);
                 const preloadedStickyShapeKeys = await archive.listDedupeCapShapeKeys();
                 const orchestrator = new CrawlerOrchestrator(archive, {
                     ...config,
                     preloadedStickyShapeKeys,
                 });
+                setupProgress?.onPhase?.(PHASE_LOADING_CRAWL_STATE);
                 const { scraped, pending } = await archive.getCrawlingState();
-                const resources = await archive.getResourceUrlList();
+                setupProgress?.onPhase?.(PHASE_LOADING_RESOURCES);
+                const resources = await archive.getResourceUrlList(setupProgress?.onChunkProgress);
+                setupProgress?.onPhase?.(PHASE_LOADING_SCRAPED_COUNT);
                 const pagesScrapedOffset = await archive.getScrapedHtmlPageCount();
+                setupProgress?.onPhase?.(PHASE_RESTORING_CRAWL_STATE);
                 orchestrator.#crawler.resume(pending, scraped, resources, pagesScrapedOffset);
                 if (initializedCallback) {
                     await initializedCallback(orchestrator, config);
                 }
                 await CrawlerOrchestrator.#preloadDnsBurnedHostCache(archive);
                 await orchestrator.crawling([], { recursive: config.recursive });
-                CrawlerOrchestrator.#finalizeCrawlSession();
-                await archive.setUrlOrder();
+                CrawlerOrchestrator.#finalizeCrawlSession(orchestrator);
+                await orchestrator.#setUrlOrder();
                 await ignoreEnoent(unlinkFile(backupPath));
                 return orchestrator;
             }
             catch (error) {
                 try {
-                    await copyFile(backupPath, absFilePath);
+                    setupProgress?.onPhase?.(RECOVERY_RESTORE_FROM_BACKUP);
+                    await copyFileWithProgress(backupPath, absFilePath, setupProgress?.onCopyProgress);
                     await ignoreEnoent(unlinkFile(backupPath));
                 }
                 catch (restoreError) {
@@ -1208,15 +1360,27 @@ export class CrawlerOrchestrator extends EventEmitter {
      * @param stubPath - Path to the existing archive file to resume from.
      * @param options - Optional configuration overrides to apply on top of the archived config.
      * @param initializedCallback - Optional callback invoked after initialization but before crawling resumes.
+     * @param setupProgress - Optional progress callbacks for the setup phase
+     *   (self-healing migrations, state rebuild) that runs before
+     *   `initializedCallback` — see {@link SetupProgressCallbacks} for why
+     *   this can't go through the orchestrator's event emitter (issue #294).
+     *   No `onExtractProgress`/`onCopyProgress`: unlike `append`/`inventory`/
+     *   `retryFailed`, `resume` reconnects to an existing tmpDir (no untar)
+     *   and takes no `.bak` (nothing to restore — the interrupted crawl's
+     *   tmpDir IS the source of truth).
      * @returns A promise that resolves to the CrawlerOrchestrator instance after crawling completes.
      * @throws {Error} If the archived URL is invalid.
      */
-    static async resume(stubPath, options, initializedCallback) {
-        const archive = await Archive.resume(stubPath);
+    static async resume(stubPath, options, initializedCallback, setupProgress) {
+        const [PHASE_RECONNECTING, PHASE_LOADING_CONFIG, PHASE_LOADING_DEDUPE_KEYS, PHASE_LOADING_CRAWL_STATE, PHASE_LOADING_RESOURCES, PHASE_LOADING_SCRAPED_COUNT, PHASE_RESTORING_CRAWL_STATE,] = RESUME_SETUP_PHASES;
+        setupProgress?.onPhase?.(PHASE_RECONNECTING);
+        const archive = await Archive.resume(stubPath, setupProgress?.onLog);
+        setupProgress?.onPhase?.(PHASE_LOADING_CONFIG);
         const archivedConfig = await archive.getConfig();
         // Seed the sticky set from prior sessions' confirmed traps so
         // `--resume` does not pay the cost of re-discovering them (see
         // `DedupeCapTracker`'s constructor JSDoc).
+        setupProgress?.onPhase?.(PHASE_LOADING_DEDUPE_KEYS);
         const preloadedStickyShapeKeys = await archive.listDedupeCapShapeKeys();
         const config = {
             ...archivedConfig,
@@ -1229,9 +1393,13 @@ export class CrawlerOrchestrator extends EventEmitter {
         if (!url) {
             throw new Error(`URL (${_url}) is invalid`);
         }
+        setupProgress?.onPhase?.(PHASE_LOADING_CRAWL_STATE);
         const { scraped, pending } = await archive.getCrawlingState();
-        const resources = await archive.getResourceUrlList();
+        setupProgress?.onPhase?.(PHASE_LOADING_RESOURCES);
+        const resources = await archive.getResourceUrlList(setupProgress?.onChunkProgress);
+        setupProgress?.onPhase?.(PHASE_LOADING_SCRAPED_COUNT);
         const pagesScrapedOffset = await archive.getScrapedHtmlPageCount();
+        setupProgress?.onPhase?.(PHASE_RESTORING_CRAWL_STATE);
         orchestrator.#crawler.resume(pending, scraped, resources, pagesScrapedOffset);
         if (initializedCallback) {
             await initializedCallback(orchestrator, config);
@@ -1242,7 +1410,7 @@ export class CrawlerOrchestrator extends EventEmitter {
         log('Config %O', config);
         await CrawlerOrchestrator.#preloadDnsBurnedHostCache(archive);
         await orchestrator.crawling([url]);
-        CrawlerOrchestrator.#finalizeCrawlSession();
+        CrawlerOrchestrator.#finalizeCrawlSession(orchestrator);
         return orchestrator;
     }
     /**
@@ -1315,21 +1483,30 @@ export class CrawlerOrchestrator extends EventEmitter {
         });
     }
     /**
-     * Tears down session-scoped crawler caches and prints a short-circuit
+     * Tears down session-scoped crawler caches and reports a short-circuit
      * summary if any URL fetches were skipped. Invoked at every
      * crawl-session boundary (`crawling` / `append` / `inventory` /
-     * `retryFailed` / `resume`).
+     * `retryFailed` / `resume`), in the same crawl-tail window as
+     * `flushingPendingWrites`/`sortingUrls` — reports through the
+     * `crawlSessionNotice` event rather than a bare `console.error` (issue
+     * #294 code review) for the same reason those two do: this runs while a
+     * caller's `Lanes`/`TaskList` display can already be active, and a
+     * direct stream write there corrupts its cursor tracking.
+     * @param orchestrator - The session's orchestrator instance, to emit
+     *   `crawlSessionNotice` from.
      */
-    static #finalizeCrawlSession() {
+    static #finalizeCrawlSession(orchestrator) {
         const skipped = dnsBurnedHostShortCircuitCounter.count;
         if (skipped > 0) {
-            // eslint-disable-next-line no-console
-            console.error(`[preload] Short-circuited ${skipped} URL(s) on DNS-burned hosts`);
+            void orchestrator.emit('crawlSessionNotice', {
+                message: `[preload] Short-circuited ${skipped} URL(s) on DNS-burned hosts`,
+            });
         }
         const { confirmedCount, totalDurationMs } = networkOutageSummaryCounter;
         if (confirmedCount > 0) {
-            // eslint-disable-next-line no-console
-            console.error(`[network] ${confirmedCount} outage(s), ${Math.round(totalDurationMs / 1000)}s total`);
+            void orchestrator.emit('crawlSessionNotice', {
+                message: `[network] ${confirmedCount} outage(s), ${Math.round(totalDurationMs / 1000)}s total`,
+            });
         }
         networkOutageSummaryCounter.confirmedCount = 0;
         networkOutageSummaryCounter.totalDurationMs = 0;