@nitpicker/crawler 0.18.1 → 0.20.0

package/lib/crawler/scan-js-resource-for-license-comment.js

@@ -0,0 +1,140 @@
+import redirects from 'follow-redirects';
+import { TECHNOLOGY_SIGNAL_DEFINITIONS } from '../archive/meta/technologies/technology-signal-definitions.js';
+/** Default cap on bytes read from a JS resource before giving up on a match. */
+const DEFAULT_BYTE_LIMIT = 8192;
+/** Default network timeout, shorter than `fetchDestination`'s HEAD budget — this is a best-effort enrichment pass, not the crawl's critical path. */
+const DEFAULT_TIMEOUT_MS = 8000;
+const EVIDENCE_MAX_LENGTH = 200;
+/** Definitions this scan tests against — only the `js-license-comment` signal type applies to JS resource bodies (the rest match HTML). */
+const JS_LICENSE_COMMENT_DEFINITIONS = TECHNOLOGY_SIGNAL_DEFINITIONS.filter((def) => def.signalType === 'js-license-comment');
+/**
+ * Reads up to `byteLimit` bytes from a JS resource's leading bytes and tests
+ * them against {@link TECHNOLOGY_SIGNAL_DEFINITIONS}' `js-license-comment`
+ * patterns (e.g. Vue's leading license-banner comment). Best-effort: any network
+ * failure, timeout, or non-2xx response resolves `null` rather than
+ * throwing — a single unreachable JS resource must not abort the enrichment
+ * pass over the rest of the archive's resources (see
+ * `scanJsResourcesForTechnologySignals`, this function's only caller).
+ *
+ * Not cached and not routed through `destinationCache` (unlike
+ * `fetchDestination`): callers are expected to persist the outcome in
+ * `technology_js_scan_cache`, keyed by `resourceId`, so a resource is never
+ * scanned twice across the archive's lifetime.
+ * @param url - The JS resource's absolute URL.
+ * @param options - Byte cap, timeout, and User-Agent overrides.
+ * @returns The first matching signal, or `null` when nothing matched (or
+ *   the fetch failed).
+ * @example
+ * const signal = await scanJsResourceForLicenseComment('https://example.com/_astro/app.js');
+ * // { technology: 'Vue', signalType: 'js-license-comment', evidence: 'Vue.js license banner text', weight: 55, category: 'JavaScript frameworks' }
+ */
+export async function scanJsResourceForLicenseComment(url, options = {}) {
+    const byteLimit = options.byteLimit ?? DEFAULT_BYTE_LIMIT;
+    const timeout = options.timeout ?? DEFAULT_TIMEOUT_MS;
+    const body = await readLeadingBytes(url, byteLimit, timeout, options.userAgent).catch(() => null);
+    if (body == null)
+        return null;
+    for (const def of JS_LICENSE_COMMENT_DEFINITIONS) {
+        const match = def.pattern.exec(body);
+        if (match) {
+            return {
+                technology: def.technology,
+                signalType: def.signalType,
+                evidence: match[0].slice(0, EVIDENCE_MAX_LENGTH),
+                weight: def.weight,
+                category: def.category ?? null,
+            };
+        }
+    }
+    return null;
+}
+/**
+ * Streams up to `byteLimit` bytes of a GET response body, destroying the
+ * connection once the cap is reached (or the stream ends first). Mirrors
+ * `fetchDestination`'s `titleBytesLimit` byte-cap technique (same
+ * `follow-redirects` + manual chunk accumulation + `req.destroy()` shape),
+ * but generic over "leading bytes as text" rather than title extraction —
+ * this module has no `PageData` to build.
+ * @param url - The absolute URL to fetch.
+ * @param byteLimit - Byte cap on the accumulated body.
+ * @param timeout - Milliseconds before the request is aborted.
+ * @param userAgent - Optional User-Agent header.
+ * @returns The accumulated body text, or `null` on a non-2xx response.
+ */
+async function readLeadingBytes(url, byteLimit, timeout, userAgent) {
+    return new Promise((resolve, reject) => {
+        const parsed = new URL(url);
+        const request = {
+            protocol: parsed.protocol,
+            hostname: parsed.hostname,
+            port: parsed.port || undefined,
+            path: `${parsed.pathname}${parsed.search}`,
+            method: 'GET',
+            trackRedirects: true,
+            headers: {
+                host: parsed.host,
+                ...(userAgent ? { 'User-Agent': userAgent } : {}),
+                Accept: '*/*',
+                'Accept-Encoding': 'identity',
+            },
+        };
+        let req;
+        let destroyed = false;
+        let settled = false;
+        const timeoutHandle = setTimeout(() => {
+            if (settled)
+                return;
+            settled = true;
+            destroyed = true;
+            req.destroy();
+            resolve(null);
+        }, timeout);
+        const response = (res) => {
+            const status = res.statusCode ?? 0;
+            if (status < 200 || status >= 300) {
+                settled = true;
+                clearTimeout(timeoutHandle);
+                destroyed = true;
+                req.destroy();
+                resolve(null);
+                return;
+            }
+            const chunks = [];
+            let totalBytes = 0;
+            res.on('data', (chunk) => {
+                if (settled)
+                    return;
+                chunks.push(chunk);
+                totalBytes += chunk.length;
+                if (totalBytes >= byteLimit) {
+                    settled = true;
+                    clearTimeout(timeoutHandle);
+                    destroyed = true;
+                    req.destroy();
+                    resolve(Buffer.concat(chunks).toString('utf8'));
+                }
+            });
+            res.on('end', () => {
+                if (settled)
+                    return;
+                settled = true;
+                clearTimeout(timeoutHandle);
+                resolve(Buffer.concat(chunks).toString('utf8'));
+            });
+        };
+        if (parsed.protocol === 'https:') {
+            req = redirects.https.request(request, response);
+        }
+        else {
+            req = redirects.http.request(request, response);
+        }
+        req.on('error', (error) => {
+            clearTimeout(timeoutHandle);
+            // Ignore errors caused by our own intentional req.destroy() above.
+            if (destroyed)
+                return;
+            reject(error);
+        });
+        req.end();
+    });
+}

package/lib/crawler/scan-js-resources-for-technology-signals.d.ts

@@ -0,0 +1,58 @@
+import type { ArchiveAccessor } from '../archive/archive-accessor.js';
+/** Options for {@link scanJsResourcesForTechnologySignals}. */
+export interface ScanJsResourcesForTechnologySignalsOptions {
+    /** Maximum concurrent network fetches. Defaults to {@link DEFAULT_CONCURRENCY}. */
+    concurrency?: number;
+    /** Forwarded to `scanJsResourceForLicenseComment`. */
+    byteLimit?: number;
+    /** Forwarded to `scanJsResourceForLicenseComment`. */
+    timeout?: number;
+    /** Forwarded to `scanJsResourceForLicenseComment`. */
+    userAgent?: string;
+    /** Called after each resource finishes scanning (matched or not). */
+    onProgress?: (done: number, total: number) => void;
+}
+/** Outcome counters returned by {@link scanJsResourcesForTechnologySignals}. */
+export interface ScanJsResourcesForTechnologySignalsResult {
+    /** JS resources eligible for scanning (internal, not yet cached). */
+    candidateCount: number;
+    /** Resources actually scanned this run (equals `candidateCount` barring a mid-run crash). */
+    scannedCount: number;
+    /** Resources whose leading bytes matched a known license comment. */
+    matchedCount: number;
+    /** Distinct pages whose `technology_signals` / `page_technologies` were updated as a result. */
+    pagesUpdatedCount: number;
+}
+/**
+ * Post-crawl network enrichment (distinct from crawl-time extraction and
+ * from read-model-time backfill — see ARCHITECTURE.md): re-fetches the
+ * leading bytes of every not-yet-scanned internal JS resource, tests them
+ * for a known technology's license comment, and folds any match into the
+ * referencing pages' `technology_signals` / `page_technologies`.
+ *
+ * Each resource is scanned at most once ever, across the archive's whole
+ * lifetime — outcomes (including non-matches) are recorded in
+ * `technology_js_scan_cache` keyed by `resourceId`, so a later
+ * `--append`/`--retry-failed` run only pays the network cost for resources
+ * discovered since the last run.
+ *
+ * A single resource can be referenced by many pages (a shared bundle); a
+ * match is applied to every one of them independently. Network scanning
+ * runs at bounded concurrency; the per-page DB recombination that follows
+ * runs after every scan has settled, never concurrently for the same page,
+ * so two resources that both resolve to the same page cannot race each
+ * other's read-modify-write.
+ *
+ * Best-effort like `scanJsResourceForLicenseComment`: an unreachable
+ * resource is recorded as a non-match (cached as scanned, `technology:
+ * null`) rather than retried or surfaced as an error — a flaky CDN must not
+ * block the rest of the archive's enrichment, and the resource will not be
+ * retried until `technology_js_scan_cache` itself is cleared.
+ * @param accessor - The archive to enrich.
+ * @param options - Concurrency, byte-cap, timeout, and progress overrides.
+ * @returns Counters describing what was scanned, matched, and updated.
+ * @example
+ * const result = await scanJsResourcesForTechnologySignals(archive);
+ * // { candidateCount: 42, scannedCount: 42, matchedCount: 3, pagesUpdatedCount: 57 }
+ */
+export declare function scanJsResourcesForTechnologySignals(accessor: ArchiveAccessor, options?: ScanJsResourcesForTechnologySignalsOptions): Promise<ScanJsResourcesForTechnologySignalsResult>;

package/lib/crawler/scan-js-resources-for-technology-signals.js

@@ -0,0 +1,196 @@
+import { combineTechnologyConfidence } from '../archive/meta/technologies/combine-technology-confidence.js';
+import { scanJsResourceForLicenseComment } from './scan-js-resource-for-license-comment.js';
+/**
+ * Content-Type strings this scan treats as JavaScript. Deliberately
+ * duplicated (not imported) from `@nitpicker/query`'s `content-type-rules.ts`
+ * `'javascript'` category rule: the crawler package must not depend back on
+ * query (see `ensure-viewer-read-model-quietly.ts`'s docs for the same
+ * boundary), and this list is small and stable enough that duplication is
+ * cheaper than introducing a shared third package for it.
+ */
+const JS_CONTENT_TYPES = [
+    'text/javascript',
+    'application/javascript',
+    'application/x-javascript',
+    'application/ecmascript',
+];
+const DEFAULT_CONCURRENCY = 4;
+/**
+ * Runs `worker` over `items` with at most `concurrency` calls in flight at
+ * once. A minimal worker-pool, not a chunk-then-parallelize helper like
+ * `eachSplitted` (which parallelizes across chunks, not within one) — this
+ * module needs a true cap on simultaneous outbound HTTP connections.
+ * @param items - The items to process.
+ * @param concurrency - Maximum simultaneous `worker` calls.
+ * @param worker - Called once per item; errors propagate to the caller.
+ */
+async function runWithConcurrency(items, concurrency, worker) {
+    let nextIndex = 0;
+    /**
+     *
+     */
+    async function runNext() {
+        const index = nextIndex++;
+        if (index >= items.length)
+            return;
+        await worker(items[index]);
+        await runNext();
+    }
+    await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => runNext()));
+}
+/**
+ * Re-derives one page's `technology_signals` + `page_technologies` rows from
+ * its existing persisted signals plus newly-discovered `js-license-comment`
+ * signals, then scoped-replaces both tables — the same full-per-page-replace
+ * invariant `insertTechnologies` follows (see its docs: the two tables are
+ * never updated independently).
+ *
+ * `technology_signals` does not persist `category`/`version` (only
+ * `page_technologies` does — see `create-adjunct-tables.ts`), so a technology
+ * whose category/version came from a signal not re-derivable from the
+ * persisted rows (a `wappalyzer` or `meta-generator` signal, both computed at
+ * crawl time from data this function does not have) would otherwise regress
+ * to `null` here. The existing `page_technologies` row is read first and its
+ * `category`/`version` fall back in wherever the freshly combined result has
+ * none.
+ * @param knex - The archive's Knex instance.
+ * @param pageId - The page to update.
+ * @param newSignals - Newly-discovered signals for this page (from JS
+ *   resources it references).
+ */
+async function applyNewSignalsToPage(knex, pageId, newSignals) {
+    await knex.transaction(async (trx) => {
+        const existingSignalRows = await trx('technology_signals')
+            .where('pageId', pageId)
+            .select('technology', 'signalType', 'evidence', 'weight');
+        const existingTechnologyRows = await trx('page_technologies')
+            .where('pageId', pageId)
+            .select('technology', 'category', 'version');
+        const existingMetaByTechnology = new Map(existingTechnologyRows.map((row) => [row.technology, row]));
+        const allSignals = [...existingSignalRows, ...newSignals];
+        const technologies = combineTechnologyConfidence(allSignals).map((t) => {
+            const previous = existingMetaByTechnology.get(t.technology);
+            return {
+                ...t,
+                category: t.category ?? previous?.category ?? null,
+                version: t.version ?? previous?.version ?? null,
+            };
+        });
+        await trx('technology_signals').where('pageId', pageId).delete();
+        await trx('page_technologies').where('pageId', pageId).delete();
+        await trx('technology_signals').insert(allSignals.map((s) => ({
+            pageId,
+            technology: s.technology,
+            signalType: s.signalType,
+            evidence: s.evidence,
+            weight: s.weight,
+        })));
+        if (technologies.length > 0) {
+            await trx('page_technologies').insert(technologies.map((t) => ({ pageId, ...t })));
+        }
+    });
+}
+/**
+ * Post-crawl network enrichment (distinct from crawl-time extraction and
+ * from read-model-time backfill — see ARCHITECTURE.md): re-fetches the
+ * leading bytes of every not-yet-scanned internal JS resource, tests them
+ * for a known technology's license comment, and folds any match into the
+ * referencing pages' `technology_signals` / `page_technologies`.
+ *
+ * Each resource is scanned at most once ever, across the archive's whole
+ * lifetime — outcomes (including non-matches) are recorded in
+ * `technology_js_scan_cache` keyed by `resourceId`, so a later
+ * `--append`/`--retry-failed` run only pays the network cost for resources
+ * discovered since the last run.
+ *
+ * A single resource can be referenced by many pages (a shared bundle); a
+ * match is applied to every one of them independently. Network scanning
+ * runs at bounded concurrency; the per-page DB recombination that follows
+ * runs after every scan has settled, never concurrently for the same page,
+ * so two resources that both resolve to the same page cannot race each
+ * other's read-modify-write.
+ *
+ * Best-effort like `scanJsResourceForLicenseComment`: an unreachable
+ * resource is recorded as a non-match (cached as scanned, `technology:
+ * null`) rather than retried or surfaced as an error — a flaky CDN must not
+ * block the rest of the archive's enrichment, and the resource will not be
+ * retried until `technology_js_scan_cache` itself is cleared.
+ * @param accessor - The archive to enrich.
+ * @param options - Concurrency, byte-cap, timeout, and progress overrides.
+ * @returns Counters describing what was scanned, matched, and updated.
+ * @example
+ * const result = await scanJsResourcesForTechnologySignals(archive);
+ * // { candidateCount: 42, scannedCount: 42, matchedCount: 3, pagesUpdatedCount: 57 }
+ */
+export async function scanJsResourcesForTechnologySignals(accessor, options = {}) {
+    const knex = accessor.getKnex();
+    const concurrency = Math.max(options.concurrency ?? DEFAULT_CONCURRENCY, 1);
+    const candidates = await knex('resource_items as ri')
+        .join('url_refs as ur', 'ur.id', 'ri.url_id')
+        .leftJoin('content_type_refs as ctr', 'ctr.id', 'ri.content_type_id')
+        .leftJoin('technology_js_scan_cache as cache', 'cache.resourceId', 'ri.id')
+        .where('ri.is_external', 0)
+        .whereNull('cache.resourceId')
+        .where((qb) => {
+        qb.whereIn('ctr.raw', JS_CONTENT_TYPES)
+            .orWhere('ur.url', 'like', '%.js')
+            .orWhere('ur.url', 'like', '%.js?%')
+            .orWhere('ur.url', 'like', '%.mjs')
+            .orWhere('ur.url', 'like', '%.mjs?%');
+    })
+        .select('ri.id as resourceId', 'ur.url as url');
+    let scannedCount = 0;
+    const matchesByResourceId = new Map();
+    await runWithConcurrency(candidates, concurrency, async (candidate) => {
+        const signal = await scanJsResourceForLicenseComment(candidate.url, {
+            byteLimit: options.byteLimit,
+            timeout: options.timeout,
+            userAgent: options.userAgent,
+        });
+        scannedCount++;
+        options.onProgress?.(scannedCount, candidates.length);
+        await knex('technology_js_scan_cache').insert({
+            resourceId: candidate.resourceId,
+            scannedAt: Date.now(),
+            technology: signal?.technology ?? null,
+            evidence: signal?.evidence ?? null,
+        });
+        if (signal) {
+            matchesByResourceId.set(candidate.resourceId, signal);
+        }
+    });
+    if (matchesByResourceId.size === 0) {
+        return {
+            candidateCount: candidates.length,
+            scannedCount,
+            matchedCount: 0,
+            pagesUpdatedCount: 0,
+        };
+    }
+    const edges = await knex('resource_ref_edges')
+        .whereIn('resource_id', [...matchesByResourceId.keys()])
+        .select('resource_id', 'page_id');
+    const newSignalsByPageId = new Map();
+    for (const edge of edges) {
+        const signal = matchesByResourceId.get(edge.resource_id);
+        if (!signal)
+            continue;
+        const list = newSignalsByPageId.get(edge.page_id);
+        if (list) {
+            list.push(signal);
+        }
+        else {
+            newSignalsByPageId.set(edge.page_id, [signal]);
+        }
+    }
+    const affectedPageIds = [...newSignalsByPageId.keys()];
+    await runWithConcurrency(affectedPageIds, concurrency, async (pageId) => {
+        await applyNewSignalsToPage(knex, pageId, newSignalsByPageId.get(pageId));
+    });
+    return {
+        candidateCount: candidates.length,
+        scannedCount,
+        matchedCount: matchesByResourceId.size,
+        pagesUpdatedCount: affectedPageIds.length,
+    };
+}

package/lib/crawler/types.d.ts

@@ -42,8 +42,18 @@ export interface RedirectEdgeResult {
  * The outcome of {@link Crawler.#scrapePage}: either a normal scrape result from
  * the browser/HEAD pipeline, or a {@link RedirectEdgeResult} when the URL's
  * redirect destination was already rendered and only the edge needs recording.
+ *
+ * The non-redirect-edge member is {@link BrowserScrapeResult}, not bare
+ * beholder `ScrapeResult` — every `#scrapePage` return site is either a
+ * `_launchBrowserAndScrape` call (which returns `BrowserScrapeResult`) or a
+ * plain `ScrapeResult`-shaped literal (structurally compatible, since
+ * `postNavigationUrl` is optional). Callers that reach a `type: 'error'`
+ * result and need to know whether the browser ended up off-host before
+ * failing (`pageData` is absent on error, so `pageData.isExternal` is not
+ * available) read `postNavigationUrl` for that signal — see
+ * `resolveResultWentOffHost`.
  */
-export type ScrapeOutcome = ScrapeResult | RedirectEdgeResult;
+export type ScrapeOutcome = BrowserScrapeResult | RedirectEdgeResult;
 /**
  * Internal envelope returned by {@link Crawler.#launchBrowserAndScrape} that
  * augments beholder's {@link ScrapeResult} with the puppeteer-side
@@ -301,6 +311,14 @@ export interface CrawlerEventTypes {
          * common case outside `crawl --inventory`. See {@link PageSource}.
          */
         source?: PageSource;
+        /**
+         * This page's body hash, precomputed by the crawler from `result.html`
+         * (non-null whenever `result.html.length > 0`, since this event is only
+         * emitted for internal pages). Forwarded through `Archive.setPage` to
+         * `update-page.ts` so `page_meta.body_hash` is written from this value
+         * instead of hashing the same html a second time.
+         */
+        bodyHash?: Buffer | null;
     };
     /**
      * Emitted when an external page (outside the crawl scope) has been scraped.
@@ -475,6 +493,21 @@ export interface CrawlerEventTypes {
         observedCount: number;
     };
 }
+/**
+ * One Web Component (custom element) found inside a page's main-content
+ * region by {@link ./collect-custom-elements.ts}. Unlike beholder's eight
+ * `MainContentsData` categories (headings/images/tables/buttons/iframes/
+ * videos/audios/canvases), this is captured independently by nitpicker
+ * itself — see {@link ./capture-custom-elements.ts} for why.
+ */
+export interface MainContentCustomElementCandidate {
+    /** The element's `nodeName` (always upper-cased in an HTML document, e.g. `MY-WIDGET`). */
+    nodeName: string;
+    /** The element's `id` attribute, or `null` when absent. */
+    elementId: string | null;
+    /** The element's class list, in DOM order. */
+    classList: string[];
+}
 /**
  * Tunables for `NetworkOutageDetector`.
  */

package/lib/crawler-orchestrator.d.ts

@@ -1,7 +1,7 @@
 import type { Config } from './archive/types.js';
 import type { NetworkProbe } from './crawler/probe-network.js';
 import type { InventoryMode } from './crawler/types.js';
-import type { CrawlEvent } from './types.js';
+import type { CrawlEvent, SetupProgressCallbacks } from './types.js';
 import type { ExURL } from '@d-zero/shared/parse-url';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
 import Archive from './archive/archive.js';
@@ -111,10 +111,13 @@ interface InventorySource {
  * writes the final archive file. It emits events defined by {@link CrawlEvent}.
  *
  * Instances are created via the static factory methods {@link CrawlerOrchestrator.crawling}
- * or {@link CrawlerOrchestrator.resume}; the constructor is private.
+ * or {@link CrawlerOrchestrator.resume}; the constructor is private. Implements
+ * `Symbol.asyncDispose` so callers can use `await using` to close the archive
+ * and reap zombie Chromium processes on scope exit instead of a manual
+ * `try`/`finally` around `archive.close()` + `garbageCollect()`.
  * @example
  * ```ts
- * const orchestrator = await CrawlerOrchestrator.crawling(['https://example.com'], { recursive: true });
+ * await using orchestrator = await CrawlerOrchestrator.crawling(['https://example.com'], { recursive: true });
  * await orchestrator.write();
  * ```
  */
@@ -125,6 +128,23 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      */
     get archive(): Archive;
     private constructor();
+    /**
+     * Enables `await using orchestrator = ...`. Closes the archive (write
+     * or remove tmpDir + release the lock, per {@link Archive.close}) and
+     * then reaps any zombie Chromium processes via {@link garbageCollect} —
+     * the same two-step teardown every CLI crawl command previously
+     * repeated by hand in a `finally` block.
+     *
+     * Relays `Archive.close()`'s recovery-write progress (issue #294) as
+     * `recoveringArchiveWrite`/`writeStep`/`writeTarProgress` — the same
+     * events `write()` emits — for the rare case where the file doesn't
+     * exist on disk yet at dispose time (e.g. an explicit `write()` call
+     * threw partway through). A CLI listener whose display is still open at
+     * that point (it hadn't yet seen `writeFileEnd`) picks these up for
+     * free; one that already tore down after the earlier failure silently
+     * drops them, same as any other post-close display update.
+     */
+    [Symbol.asyncDispose](): Promise<void>;
     /**
      * Abort the current crawl operation.
      *
@@ -173,8 +193,11 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * the time `write()` is called those tables are already populated.
      * This method just tars.
      *
-     * Emits `writeFileStart` before writing and `writeFileEnd` after
-     * the write completes successfully.
+     * Emits `writeFileStart` before writing and `writeFileEnd` after the
+     * write completes successfully. Also relays `Archive.write()`'s
+     * per-step (`writeStep`) and tar-byte (`writeTarProgress`) progress
+     * (issue #294) — tarring a 15 GB+ archive can take minutes, and without
+     * these events a CLI listener has no way to show it isn't hung.
      */
     write(): Promise<void>;
     /**
@@ -211,10 +234,14 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * @param newUrls - New root URLs to add and crawl.
      * @param options - Optional config overrides applied on top of the archived config.
      * @param initializedCallback - Optional callback invoked after initialization but before crawling resumes.
+     * @param setupProgress - Optional progress callbacks for the setup phase
+     *   (untar, `.bak` copy, repromote, state rebuild) that runs before
+     *   `initializedCallback` — see {@link SetupProgressCallbacks} for why
+     *   this can't go through the orchestrator's event emitter (issue #294).
      * @returns The orchestrator instance after the append crawl completes.
      * @throws {Error} When `newUrls` is empty, the archive is in list mode, or it cannot be parsed.
      */
-    static append(archivePath: string, newUrls: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback): Promise<CrawlerOrchestrator>;
+    static append(archivePath: string, newUrls: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback, setupProgress?: SetupProgressCallbacks): Promise<CrawlerOrchestrator>;
     /**
      * Inventory mode: cross-reference a user-supplied URL list against an
      * existing `.nitpicker` archive and import ONLY the URLs that are not yet
@@ -292,10 +319,15 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      *   URLs. Pass `null` for programmatic callers that built
      *   `inventoryUrls` in-memory; the audit row's `source_file_sha256`
      *   column will be `NULL` and no source list is archived.
+     * @param setupProgress - Optional progress callbacks for the setup phase
+     *   (untar, scope classification, bulk inserts, state rebuild) that runs
+     *   before `initializedCallback` — see {@link SetupProgressCallbacks} for
+     *   why this can't go through the orchestrator's event emitter (issue
+     *   #294).
      * @returns The orchestrator instance after a successful inventory pass.
      * @throws {Error} When `inventoryUrls` is empty or the archive is in list mode. Unresolved pending URLs from a previous crawl do NOT throw — see step 3.
      */
-    static inventory(archivePath: string, inventoryUrls: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback, source?: InventorySource | null): Promise<CrawlerOrchestrator>;
+    static inventory(archivePath: string, inventoryUrls: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback, source?: InventorySource | null, setupProgress?: SetupProgressCallbacks): Promise<CrawlerOrchestrator>;
     /**
      * Re-fetch previously-failed pages in an existing `.nitpicker` archive.
      *
@@ -326,10 +358,14 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * @param archivePath - Absolute or relative path to the existing `.nitpicker`.
      * @param options - Optional config overrides applied on top of the archived config.
      * @param initializedCallback - Optional callback invoked after initialization but before crawling resumes.
+     * @param setupProgress - Optional progress callbacks for the setup phase
+     *   (untar, `.bak` copy, reset, state rebuild) that runs before
+     *   `initializedCallback` — see {@link SetupProgressCallbacks} for why
+     *   this can't go through the orchestrator's event emitter (issue #294).
      * @returns The orchestrator instance after the retry crawl completes.
      * @throws {Error} When the archive is in list mode or has no parseable roots.
      */
-    static retryFailed(archivePath: string, options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback): Promise<CrawlerOrchestrator>;
+    static retryFailed(archivePath: string, options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback, setupProgress?: SetupProgressCallbacks): Promise<CrawlerOrchestrator>;
     /**
      * Resume a previously interrupted crawl from an existing archive file.
      *
@@ -339,9 +375,17 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * @param stubPath - Path to the existing archive file to resume from.
      * @param options - Optional configuration overrides to apply on top of the archived config.
      * @param initializedCallback - Optional callback invoked after initialization but before crawling resumes.
+     * @param setupProgress - Optional progress callbacks for the setup phase
+     *   (self-healing migrations, state rebuild) that runs before
+     *   `initializedCallback` — see {@link SetupProgressCallbacks} for why
+     *   this can't go through the orchestrator's event emitter (issue #294).
+     *   No `onExtractProgress`/`onCopyProgress`: unlike `append`/`inventory`/
+     *   `retryFailed`, `resume` reconnects to an existing tmpDir (no untar)
+     *   and takes no `.bak` (nothing to restore — the interrupted crawl's
+     *   tmpDir IS the source of truth).
      * @returns A promise that resolves to the CrawlerOrchestrator instance after crawling completes.
      * @throws {Error} If the archived URL is invalid.
      */
-    static resume(stubPath: string, options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback): Promise<CrawlerOrchestrator>;
+    static resume(stubPath: string, options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback, setupProgress?: SetupProgressCallbacks): Promise<CrawlerOrchestrator>;
 }
 export {};