@nitpicker/crawler 0.18.1 → 0.19.0

package/lib/crawler.d.ts

@@ -23,10 +23,11 @@ export { listArchiveCacheEntries } from './archive/cache/list-archive-cache-entr
 export { clearArchiveCacheRoot } from './archive/cache/clear-archive-cache-root.js';
 export { clearArchiveCacheEntry } from './archive/cache/clear-archive-cache-entry.js';
 export type { ArchiveCacheEntry, ArchiveCacheEntryKind } from './archive/cache/types.js';
+export { copyFileWithProgress } from './archive/filesystem/copy-file-with-progress.js';
 export { acquireArchiveLock, ArchiveLockError } from './archive/archive-lock.js';
 export { peekArchiveLockHolder } from './archive/peek-archive-lock.js';
 export type { ArchiveLockHolder } from './archive/peek-archive-lock.js';
-export type { FlatPageMetaColumns, JsonLdRow, JsonLdRowForInsert, TagRow, TagRowForInsert, JsonLdSummary, TagsSummary, TagInventoryEntry, PageDenormalizedColumns, } from './archive/meta/types.js';
+export type { FlatPageMetaColumns, JsonLdRow, JsonLdRowForInsert, TagRow, TagRowForInsert, JsonLdSummary, PageDenormalizedColumns, TechnologySignalRow, TechnologySignalRowForInsert, PageTechnologyRow, PageTechnologyRowForInsert, } from './archive/meta/types.js';
 export { IncompatibleArchiveError } from './archive/meta/types.js';
 export { REQUIRED_FORMAT_VERSION } from './archive/meta/assert-compatible-version.js';
 export { computeBodyHash } from './archive/body-hash/compute-body-hash.js';
@@ -35,6 +36,11 @@ export { computeTierAAliasKey } from './archive/url-alias/compute-tier-a-alias-k
 export { computeTierBAliasKey } from './archive/url-alias/compute-tier-b-alias-key.js';
 export { computeShapeKey } from './crawler/dedupe/compute-shape-key.js';
 export { DEFAULT_EXCLUDED_EXTERNAL_URLS, CrawlerOrchestrator, } from './crawler-orchestrator.js';
+export { RESUME_SETUP_PHASES } from './resume-setup-phases.js';
+export { APPEND_SETUP_PHASES } from './append-setup-phases.js';
+export { RETRY_FAILED_SETUP_PHASES } from './retry-failed-setup-phases.js';
+export { INVENTORY_SETUP_PHASES } from './inventory-setup-phases.js';
+export { SETUP_RECOVERY_PHASE_LABELS } from './setup-recovery-phase-labels.js';
 export * from './types.js';
 export * from './crawler/types.js';
 export { classifyErrorKind } from './classify-error-kind.js';
@@ -48,7 +54,11 @@ export { probeNetwork } from './crawler/probe-network.js';
 export { computeOutageClampTimestamp } from './archive/db-ops/outages/compute-outage-clamp-timestamp.js';
 export { chooseProbeHost } from './crawler/choose-probe-host.js';
 export { assertChromeIsInstalled } from './crawler/assert-chrome-installed.js';
+export { assertPuppeteerSharedWithBeholder } from './crawler/assert-puppeteer-shared-with-beholder.js';
 export { computeFileSha256 } from './utils/compute-file-sha256.js';
+export { scanJsResourceForLicenseComment } from './crawler/scan-js-resource-for-license-comment.js';
+export type { ScanJsResourcesForTechnologySignalsOptions, ScanJsResourcesForTechnologySignalsResult, } from './crawler/scan-js-resources-for-technology-signals.js';
+export { scanJsResourcesForTechnologySignals } from './crawler/scan-js-resources-for-technology-signals.js';
 export { populateEntityTables } from './archive/populate-entity-tables/populate-entities.js';
 export type { PageDomPathResolver } from './archive/populate-entity-tables/populate-image-items.js';
 export { populateRefTables } from './archive/populate-ref-tables/populate-refs.js';

package/lib/crawler.js

@@ -23,6 +23,7 @@ export { resolveArchiveCacheDir } from './archive/cache/resolve-archive-cache-di
 export { listArchiveCacheEntries } from './archive/cache/list-archive-cache-entries.js';
 export { clearArchiveCacheRoot } from './archive/cache/clear-archive-cache-root.js';
 export { clearArchiveCacheEntry } from './archive/cache/clear-archive-cache-entry.js';
+export { copyFileWithProgress } from './archive/filesystem/copy-file-with-progress.js';
 export { acquireArchiveLock, ArchiveLockError } from './archive/archive-lock.js';
 export { peekArchiveLockHolder } from './archive/peek-archive-lock.js';
 export { IncompatibleArchiveError } from './archive/meta/types.js';
@@ -34,6 +35,11 @@ export { computeTierBAliasKey } from './archive/url-alias/compute-tier-b-alias-k
 export { computeShapeKey } from './crawler/dedupe/compute-shape-key.js';
 // Core
 export { DEFAULT_EXCLUDED_EXTERNAL_URLS, CrawlerOrchestrator, } from './crawler-orchestrator.js';
+export { RESUME_SETUP_PHASES } from './resume-setup-phases.js';
+export { APPEND_SETUP_PHASES } from './append-setup-phases.js';
+export { RETRY_FAILED_SETUP_PHASES } from './retry-failed-setup-phases.js';
+export { INVENTORY_SETUP_PHASES } from './inventory-setup-phases.js';
+export { SETUP_RECOVERY_PHASE_LABELS } from './setup-recovery-phase-labels.js';
 export * from './types.js';
 export * from './crawler/types.js';
 export { classifyErrorKind } from './classify-error-kind.js';
@@ -45,7 +51,10 @@ export { probeNetwork } from './crawler/probe-network.js';
 export { computeOutageClampTimestamp } from './archive/db-ops/outages/compute-outage-clamp-timestamp.js';
 export { chooseProbeHost } from './crawler/choose-probe-host.js';
 export { assertChromeIsInstalled } from './crawler/assert-chrome-installed.js';
+export { assertPuppeteerSharedWithBeholder } from './crawler/assert-puppeteer-shared-with-beholder.js';
 export { computeFileSha256 } from './utils/compute-file-sha256.js';
+export { scanJsResourceForLicenseComment } from './crawler/scan-js-resource-for-license-comment.js';
+export { scanJsResourcesForTechnologySignals } from './crawler/scan-js-resources-for-technology-signals.js';
 // 0.13 ref-table population (issue #191, epic #103). Exposed as the
 // public seam that the migration script (`scripts/migrate-to-0.13.mjs`)
 // drives against an already-connected archive.

package/lib/inventory-setup-phases.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Ordered `SetupProgressCallbacks.onPhase` labels
+ * `CrawlerOrchestrator.inventory` announces, in call order, before its
+ * `initializedCallback` fires. Single source of truth for both the `onPhase`
+ * call sites in `crawler-orchestrator.ts` and the CLI's setup task list
+ * (`create-setup-task-list.ts`), which pre-builds one task-list row per
+ * entry — duplicating this sequence by hand at the CLI layer would silently
+ * drift the moment a phase is added, renamed, or reordered here.
+ *
+ * This is the **superset** sequence — `inventory` has three
+ * `initializedCallback` call sites, each reached after a different prefix of
+ * this list depending on the input:
+ * - no novel URLs (everything already known / out of scope): stops after
+ *   index 3 (`'Checking for already-known URLs'`), skips the `.bak` backup
+ *   entirely
+ * - novel URLs are all non-HTML: stops after index 7
+ *   (`'Recording excluded pages'`) — nothing to render, no crawl state to
+ *   rebuild
+ * - at least one novel HTML seed: runs the full 12-phase sequence
+ *
+ * The CLI's setup task list pre-builds all 12 rows and marks whichever
+ * suffix never got announced as skipped once `initializedCallback` fires —
+ * it does not need to know in advance which of the three prefixes a given
+ * run will take. Note `'Loading crawl state'` appears twice (indices 2 and
+ * 8): the first read is scoped to the pre-ingestion pending-URL warning, the
+ * second re-reads state after the new inventory seeds have been inserted.
+ *
+ * On failure, a `SETUP_RECOVERY_PHASE_LABELS` entry
+ * (`'Restoring archive from backup'` or `'Persisting ingested inventory
+ * state'`) is announced instead of continuing this sequence.
+ */
+export declare const INVENTORY_SETUP_PHASES: readonly ["Extracting archive", "Loading archive config", "Loading crawl state", "Checking for already-known URLs", "Backing up archive", "Recording non-HTML resources", "Recording HTML seed pages", "Recording excluded pages", "Loading crawl state", "Loading resource list", "Loading scraped page count", "Restoring crawl state"];

package/lib/inventory-setup-phases.js

@@ -0,0 +1,45 @@
+/**
+ * Ordered `SetupProgressCallbacks.onPhase` labels
+ * `CrawlerOrchestrator.inventory` announces, in call order, before its
+ * `initializedCallback` fires. Single source of truth for both the `onPhase`
+ * call sites in `crawler-orchestrator.ts` and the CLI's setup task list
+ * (`create-setup-task-list.ts`), which pre-builds one task-list row per
+ * entry — duplicating this sequence by hand at the CLI layer would silently
+ * drift the moment a phase is added, renamed, or reordered here.
+ *
+ * This is the **superset** sequence — `inventory` has three
+ * `initializedCallback` call sites, each reached after a different prefix of
+ * this list depending on the input:
+ * - no novel URLs (everything already known / out of scope): stops after
+ *   index 3 (`'Checking for already-known URLs'`), skips the `.bak` backup
+ *   entirely
+ * - novel URLs are all non-HTML: stops after index 7
+ *   (`'Recording excluded pages'`) — nothing to render, no crawl state to
+ *   rebuild
+ * - at least one novel HTML seed: runs the full 12-phase sequence
+ *
+ * The CLI's setup task list pre-builds all 12 rows and marks whichever
+ * suffix never got announced as skipped once `initializedCallback` fires —
+ * it does not need to know in advance which of the three prefixes a given
+ * run will take. Note `'Loading crawl state'` appears twice (indices 2 and
+ * 8): the first read is scoped to the pre-ingestion pending-URL warning, the
+ * second re-reads state after the new inventory seeds have been inserted.
+ *
+ * On failure, a `SETUP_RECOVERY_PHASE_LABELS` entry
+ * (`'Restoring archive from backup'` or `'Persisting ingested inventory
+ * state'`) is announced instead of continuing this sequence.
+ */
+export const INVENTORY_SETUP_PHASES = [
+    'Extracting archive',
+    'Loading archive config',
+    'Loading crawl state',
+    'Checking for already-known URLs',
+    'Backing up archive',
+    'Recording non-HTML resources',
+    'Recording HTML seed pages',
+    'Recording excluded pages',
+    'Loading crawl state',
+    'Loading resource list',
+    'Loading scraped page count',
+    'Restoring crawl state',
+];

package/lib/resume-setup-phases.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Ordered `SetupProgressCallbacks.onPhase` labels `CrawlerOrchestrator.resume`
+ * announces, in call order, before its `initializedCallback` fires. Single
+ * source of truth for both the `onPhase` call sites in
+ * `crawler-orchestrator.ts` and the CLI's setup task list (`create-setup-task-list.ts`),
+ * which pre-builds one task-list row per entry — duplicating this sequence by
+ * hand at the CLI layer would silently drift the moment a phase is added,
+ * renamed, or reordered here.
+ *
+ * `resume` never takes a `.bak` backup (there is nothing to restore — the
+ * interrupted crawl's tmp dir is the source of truth), so unlike
+ * `append`/`inventory`/`retryFailed` this sequence has no failure-only
+ * recovery phase and no branching: every `resume` call announces exactly
+ * these seven phases in this order.
+ */
+export declare const RESUME_SETUP_PHASES: readonly ["Reconnecting to archive", "Loading archive config", "Loading dedupe-cap shape keys", "Loading crawl state", "Loading resource list", "Loading scraped page count", "Restoring crawl state"];

package/lib/resume-setup-phases.js

@@ -0,0 +1,24 @@
+/**
+ * Ordered `SetupProgressCallbacks.onPhase` labels `CrawlerOrchestrator.resume`
+ * announces, in call order, before its `initializedCallback` fires. Single
+ * source of truth for both the `onPhase` call sites in
+ * `crawler-orchestrator.ts` and the CLI's setup task list (`create-setup-task-list.ts`),
+ * which pre-builds one task-list row per entry — duplicating this sequence by
+ * hand at the CLI layer would silently drift the moment a phase is added,
+ * renamed, or reordered here.
+ *
+ * `resume` never takes a `.bak` backup (there is nothing to restore — the
+ * interrupted crawl's tmp dir is the source of truth), so unlike
+ * `append`/`inventory`/`retryFailed` this sequence has no failure-only
+ * recovery phase and no branching: every `resume` call announces exactly
+ * these seven phases in this order.
+ */
+export const RESUME_SETUP_PHASES = [
+    'Reconnecting to archive',
+    'Loading archive config',
+    'Loading dedupe-cap shape keys',
+    'Loading crawl state',
+    'Loading resource list',
+    'Loading scraped page count',
+    'Restoring crawl state',
+];

package/lib/retry-failed-setup-phases.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Ordered `SetupProgressCallbacks.onPhase` labels
+ * `CrawlerOrchestrator.retryFailed` announces, in call order, before its
+ * `initializedCallback` fires. Single source of truth for both the `onPhase`
+ * call sites in `crawler-orchestrator.ts` and the CLI's setup task list
+ * (`create-setup-task-list.ts`), which pre-builds one task-list row per
+ * entry — duplicating this sequence by hand at the CLI layer would silently
+ * drift the moment a phase is added, renamed, or reordered here. Shares its
+ * prefix/suffix with `APPEND_SETUP_PHASES` via
+ * `APPEND_RETRY_FAILED_COMMON_SETUP_PHASES` — `'Resetting failed pages'` is
+ * the one step unique to `retryFailed()`.
+ *
+ * Linear on the success path (no branching before `initializedCallback`). On
+ * failure — anywhere from `archive.resetFailedPages` through the crawl
+ * itself — a `SETUP_RECOVERY_PHASE_LABELS` entry
+ * (`'Restoring archive from backup'`) is announced instead of continuing
+ * this sequence.
+ */
+export declare const RETRY_FAILED_SETUP_PHASES: readonly ["Extracting archive", "Loading archive config", "Backing up archive", "Resetting failed pages", "Loading dedupe-cap shape keys", "Loading crawl state", "Loading resource list", "Loading scraped page count", "Restoring crawl state"];

package/lib/retry-failed-setup-phases.js

@@ -0,0 +1,24 @@
+import { APPEND_RETRY_FAILED_COMMON_SETUP_PHASES } from './append-retry-failed-common-setup-phases.js';
+/**
+ * Ordered `SetupProgressCallbacks.onPhase` labels
+ * `CrawlerOrchestrator.retryFailed` announces, in call order, before its
+ * `initializedCallback` fires. Single source of truth for both the `onPhase`
+ * call sites in `crawler-orchestrator.ts` and the CLI's setup task list
+ * (`create-setup-task-list.ts`), which pre-builds one task-list row per
+ * entry — duplicating this sequence by hand at the CLI layer would silently
+ * drift the moment a phase is added, renamed, or reordered here. Shares its
+ * prefix/suffix with `APPEND_SETUP_PHASES` via
+ * `APPEND_RETRY_FAILED_COMMON_SETUP_PHASES` — `'Resetting failed pages'` is
+ * the one step unique to `retryFailed()`.
+ *
+ * Linear on the success path (no branching before `initializedCallback`). On
+ * failure — anywhere from `archive.resetFailedPages` through the crawl
+ * itself — a `SETUP_RECOVERY_PHASE_LABELS` entry
+ * (`'Restoring archive from backup'`) is announced instead of continuing
+ * this sequence.
+ */
+export const RETRY_FAILED_SETUP_PHASES = [
+    ...APPEND_RETRY_FAILED_COMMON_SETUP_PHASES.prefix,
+    'Resetting failed pages',
+    ...APPEND_RETRY_FAILED_COMMON_SETUP_PHASES.suffix,
+];

package/lib/setup-recovery-phase-labels.d.ts

@@ -0,0 +1,21 @@
+/**
+ * `SetupProgressCallbacks.onPhase` labels announced only from a failure
+ * path — never part of the ordered success-path sequences
+ * (`RESUME_SETUP_PHASES` / `APPEND_SETUP_PHASES` / `RETRY_FAILED_SETUP_PHASES` /
+ * `INVENTORY_SETUP_PHASES`) those static methods otherwise announce in order.
+ *
+ * The CLI's setup task list treats any `onPhase` label in this set as
+ * unplanned: instead of advancing to the next pre-built row, it inserts a
+ * new row for it (`ctx.insertNext`) so the recovery copy's byte progress
+ * stays visible, rather than trying to guess in advance where in the
+ * success-path sequence a failure might interrupt it.
+ *
+ * - `'Restoring archive from backup'` — `append` / `inventory` /
+ *   `retryFailed` all take this path when anything from the post-backup
+ *   setup step through the crawl itself throws.
+ * - `'Persisting ingested inventory state'` — `inventory`-only: once its
+ *   pre-insert + audit-row write has committed, a later failure (state
+ *   rebuild or the crawl itself) must persist the ingested rows to disk
+ *   instead of restoring the pre-inventory `.bak`, or they would be lost.
+ */
+export declare const SETUP_RECOVERY_PHASE_LABELS: readonly ["Restoring archive from backup", "Persisting ingested inventory state"];

package/lib/setup-recovery-phase-labels.js

@@ -0,0 +1,24 @@
+/**
+ * `SetupProgressCallbacks.onPhase` labels announced only from a failure
+ * path — never part of the ordered success-path sequences
+ * (`RESUME_SETUP_PHASES` / `APPEND_SETUP_PHASES` / `RETRY_FAILED_SETUP_PHASES` /
+ * `INVENTORY_SETUP_PHASES`) those static methods otherwise announce in order.
+ *
+ * The CLI's setup task list treats any `onPhase` label in this set as
+ * unplanned: instead of advancing to the next pre-built row, it inserts a
+ * new row for it (`ctx.insertNext`) so the recovery copy's byte progress
+ * stays visible, rather than trying to guess in advance where in the
+ * success-path sequence a failure might interrupt it.
+ *
+ * - `'Restoring archive from backup'` — `append` / `inventory` /
+ *   `retryFailed` all take this path when anything from the post-backup
+ *   setup step through the crawl itself throws.
+ * - `'Persisting ingested inventory state'` — `inventory`-only: once its
+ *   pre-insert + audit-row write has committed, a later failure (state
+ *   rebuild or the crawl itself) must persist the ingested rows to disk
+ *   instead of restoring the pre-inventory `.bak`, or they would be lost.
+ */
+export const SETUP_RECOVERY_PHASE_LABELS = [
+    'Restoring archive from backup',
+    'Persisting ingested inventory state',
+];

package/lib/types.d.ts

@@ -1,4 +1,75 @@
+import type { APPEND_SETUP_PHASES } from './append-setup-phases.js';
+import type { INVENTORY_SETUP_PHASES } from './inventory-setup-phases.js';
+import type { RESUME_SETUP_PHASES } from './resume-setup-phases.js';
+import type { RETRY_FAILED_SETUP_PHASES } from './retry-failed-setup-phases.js';
+import type { SETUP_RECOVERY_PHASE_LABELS } from './setup-recovery-phase-labels.js';
 import type { CrawlerError, PageData } from './utils/types/types.js';
+/**
+ * Every label `SetupProgressCallbacks.onPhase` can be called with, across
+ * all four `CrawlerOrchestrator` setup sequences plus the failure-only
+ * recovery phases. See `RESUME_SETUP_PHASES` / `APPEND_SETUP_PHASES` /
+ * `RETRY_FAILED_SETUP_PHASES` / `INVENTORY_SETUP_PHASES` /
+ * `SETUP_RECOVERY_PHASE_LABELS` for what each label means and when it fires.
+ */
+export type SetupPhaseLabel = (typeof RESUME_SETUP_PHASES)[number] | (typeof APPEND_SETUP_PHASES)[number] | (typeof RETRY_FAILED_SETUP_PHASES)[number] | (typeof INVENTORY_SETUP_PHASES)[number] | (typeof SETUP_RECOVERY_PHASE_LABELS)[number];
+/**
+ * Progress callbacks for the setup phase of `CrawlerOrchestrator.append` /
+ * `inventory` / `retryFailed` / `resume` (issue #294) — everything from
+ * `Archive.open`'s untar through `Crawler#resume`'s in-memory state rebuild,
+ * which all runs **before** `initializedCallback` fires (before the CLI's
+ * event-based progress display — `attachCrawlDisplay` — has anything to
+ * subscribe to). A large archive's setup can itself take tens of seconds to
+ * minutes (untar, `.bak` copy, chunked page/resource re-scans), and without
+ * this it looked completely silent — including before the CLI's own
+ * "🐳 archive (...)" header line, which normally establishes that the
+ * process is even alive.
+ *
+ * Passed as a plain callback object (not routed through the orchestrator's
+ * event emitter) because the orchestrator instance the emitter lives on
+ * does not exist yet for most of this phase — `Archive.open`, the `.bak`
+ * copy, and the pre-`new CrawlerOrchestrator(...)` scope/repromote work all
+ * run before there is anything to attach a listener to.
+ */
+export interface SetupProgressCallbacks {
+    /**
+     * Called once at the start of each named setup step that has no
+     * countable progress of its own (a single query, an in-memory rebuild).
+     * @param label - Human-readable description of the step starting.
+     */
+    onPhase?: (label: SetupPhaseLabel) => void;
+    /**
+     * Called during `Archive.open`'s tar extraction, with bytes read so far
+     * and the archive's total size.
+     */
+    onExtractProgress?: (readBytes: number, totalBytes: number) => void;
+    /**
+     * Called during a `.bak` backup or restore copy, with bytes copied so
+     * far and the source file's total size — the pre-mutation backup and
+     * the on-failure restore both go through this one callback; the
+     * caller's own {@link SetupProgressCallbacks.onPhase} call immediately
+     * before distinguishes which copy is running.
+     */
+    onCopyProgress?: (copiedBytes: number, totalBytes: number) => void;
+    /**
+     * Called for chunked/keyset-scan progress within whichever named phase
+     * is currently running (`repromoteExternalPages`, `getResourceUrlList`,
+     * `resetFailedPages`) — the unit varies by phase (pages processed vs.
+     * ids scanned), so callers label it using the most recent `onPhase`
+     * call.
+     * @param processed - Units completed so far, including this update.
+     * @param total - Total units this phase will process.
+     */
+    onChunkProgress?: (processed: number, total: number) => void;
+    /**
+     * Called instead of `console.error` for self-healing schema migration
+     * notices that can fire during `Archive.open`/`Archive.resume` while
+     * this setup phase's own `Lanes`/`TaskList` display is active (issue
+     * #294) — a bare `console.error` there corrupts the display's cursor
+     * tracking. Forwarded to {@link import('../archive/archive.js').ArchiveOpenOptions.onLog}.
+     * Omit to fall back to `console.error`.
+     */
+    onLog?: (message: string) => void;
+}
 /**
  * Aggregate counts captured during a `--inventory` invocation, forwarded to
  * `#writeInventoryRunRow` so the audit log row is consistent between the
@@ -115,6 +186,80 @@ export interface CrawlEvent {
         /** Absolute path of the archive file that was written. */
         filePath: string;
     };
+    /**
+     * Emitted once at the start of each of `Archive.write()`'s internal
+     * steps (issue #294) — `checkpoint`/`remove` have no countable progress
+     * of their own (a single synchronous PRAGMA and a directory removal),
+     * so without this a large archive's write looks frozen between
+     * `writeFileStart` and the `writeTarProgress` byte updates.
+     */
+    writeStep: {
+        /** Which step of `Archive.write()` is starting. */
+        step: 'checkpoint' | 'rename' | 'tar' | 'remove';
+    };
+    /**
+     * Emitted as archive bytes are written during `Archive.write()`'s tar
+     * step (issue #294) — tarring a large (15 GB+) archive can take minutes,
+     * and without this the CLI shows nothing between `writeStep: 'tar'` and
+     * `writeFileEnd`.
+     */
+    writeTarProgress: {
+        /** Bytes written so far. */
+        writtenBytes: number;
+        /** Estimated total bytes (sum of source file sizes; tar adds headers/padding). */
+        totalBytes: number;
+    };
+    /**
+     * Emitted as pages are re-ordered after crawling completes (issue #294)
+     * — `setUrlOrder()` loads every internal page, sorts it in JS, then
+     * writes the result back in chunks, which can take seconds to minutes
+     * on a large archive with no other signal it hasn't hung.
+     */
+    sortingUrls: {
+        /** Pages assigned an order so far. */
+        processed: number;
+        /** Total internal pages being ordered. */
+        total: number;
+    };
+    /**
+     * Emitted once, only when `[Symbol.asyncDispose]`'s call to
+     * `Archive.close()` discovers the archive file doesn't exist on disk
+     * yet and falls back to writing it there (issue #294) — e.g. the
+     * caller's own explicit `CrawlerOrchestrator.write()` threw partway
+     * through (network/disk error during tar) before finishing. Emitted
+     * before the `writeStep`/`writeTarProgress` events this recovery write
+     * reuses, so a listener knows why a fresh write is starting after one
+     * already appeared to fail.
+     */
+    recoveringArchiveWrite: Record<string, never>;
+    /**
+     * Emitted once, right before `crawlEnd`'s final `WriteQueue.drain()`,
+     * when that queue still has enqueued-but-not-yet-executed writes (issue
+     * #294) — the deal's own per-page progress display has already stopped
+     * updating by this point (the crawler itself is done), so a queue still
+     * draining page/resource INSERTs would otherwise look like the process
+     * hung between the last progress line and the archive write starting.
+     * Not emitted when the queue is already empty — nothing to wait on, so
+     * nothing to explain.
+     */
+    flushingPendingWrites: {
+        /** Enqueued operations still waiting or executing at emission time. */
+        pending: number;
+    };
+    /**
+     * Emitted once per session-summary notice `#finalizeCrawlSession` has to
+     * report — the DNS-burned-host short-circuit count and/or the
+     * network-outage summary, each only when its count is nonzero (issue
+     * #294 code review). Fires in the same crawl-tail window as
+     * `flushingPendingWrites`/`sortingUrls` (after crawling finishes, before
+     * the static factory method returns), so a listener can route it the
+     * same way instead of a bare `console.error` corrupting whatever
+     * `Lanes`/`TaskList` display happens to be active at that point.
+     */
+    crawlSessionNotice: {
+        /** The formatted, ready-to-display notice text. */
+        message: string;
+    };
     /**
      * Emitted when an error occurs during crawling or archiving.
      */

package/package.json

@@ -1,9 +1,12 @@
 {
 	"name": "@nitpicker/crawler",
-	"version": "0.18.1",
+	"version": "0.19.0",
 	"description": "Web crawler engine with headless browser rendering and archive storage",
 	"author": "D-ZERO",
 	"license": "Apache-2.0",
+	"engines": {
+		"node": ">=24.11.0"
+	},
 	"repository": {
 		"type": "git",
 		"url": "https://github.com/d-zero-dev/nitpicker.git",
@@ -27,17 +30,17 @@
 		"clean": "tsc --build --clean"
 	},
 	"dependencies": {
-		"@d-zero/beholder": "4.2.2",
-		"@d-zero/dealer": "1.10.4",
-		"@d-zero/fs": "0.2.6",
-		"@d-zero/shared": "0.22.5",
+		"@d-zero/beholder": "4.2.3",
+		"@d-zero/dealer": "1.12.0",
+		"@d-zero/fs": "0.2.7",
+		"@d-zero/shared": "0.23.0",
 		"ansi-colors": "4.1.3",
 		"debug": "4.4.3",
 		"follow-redirects": "1.16.0",
-		"fs-extra": "11.3.6",
+		"fs-extra": "11.4.0",
 		"knex": "3.3.0",
 		"libsql": "0.5.29",
-		"puppeteer": "25.3.0",
+		"puppeteer": "25.5.0",
 		"robots-parser": "3.0.1",
 		"tar": "7.5.22"
 	},
@@ -48,5 +51,5 @@
 		"@types/tar": "7.0.87",
 		"@types/unzipper": "0.10.11"
 	},
-	"gitHead": "cf366bfa2ca1e0528623b3fbe1c17ac4237ca4e4"
+	"gitHead": "b79b5c304820d924168a139e7148e0593c29fe1c"
 }

package/lib/archive/db-ops/meta/get-tags-of-page.d.ts

@@ -1,12 +0,0 @@
-import type { TagRow } from '../../meta/types.js';
-import type { Knex } from 'knex';
-/**
- * Retrieves all `page_tags` rows for the given page id, parsed back into
- * {@link TagRow} shape (with `categories` and `sources` JSON columns
- * deserialised).
- *
- * Read-side counterpart to `insertTags`.
- * @param knex - Knex query builder connected to the archive DB.
- * @param pageId
- */
-export declare function getTagsOfPage(knex: Knex, pageId: number): Promise<TagRow[]>;

package/lib/archive/db-ops/meta/get-tags-of-page.js

@@ -1,28 +0,0 @@
-import { safeParseJson } from '../_shared/safe-parse-json.js';
-/**
- * Retrieves all `page_tags` rows for the given page id, parsed back into
- * {@link TagRow} shape (with `categories` and `sources` JSON columns
- * deserialised).
- *
- * Read-side counterpart to `insertTags`.
- * @param knex - Knex query builder connected to the archive DB.
- * @param pageId
- */
-export async function getTagsOfPage(knex, pageId) {
-    const rows = await knex
-        .select('id', 'pageId', 'provider', 'category', 'externalId', 'version', 'confidence', 'categories', 'sources')
-        .from('page_tags')
-        .where('pageId', pageId)
-        .orderBy('id', 'asc');
-    return rows.map((r) => ({
-        id: r.id,
-        pageId: r.pageId,
-        provider: r.provider,
-        category: r.category,
-        externalId: r.externalId,
-        version: r.version,
-        confidence: r.confidence,
-        categories: r.categories === null ? [] : (safeParseJson(r.categories) ?? []),
-        sources: r.sources === null ? [] : (safeParseJson(r.sources) ?? []),
-    }));
-}

package/lib/archive/db-ops/pages/write/insert-tags.d.ts

@@ -1,16 +0,0 @@
-import type { PageData } from '../../../../utils/types/types.js';
-import type { Knex } from 'knex';
-/**
- * Replaces the page's Wappalyzer tag rows with the freshly captured set.
- * Called inside `updatePage`'s transaction unconditionally — tag
- * detection draws on `<script src>` / `<iframe src>` / window globals /
- * response headers, not the HTML body, so external pages that skip
- * rendering still contribute tags.
- *
- * Same empty-guard as `insertJsonLd`: an empty array does not wipe
- * prior rows on a degraded re-scrape.
- * @param pageId
- * @param meta
- * @param trx
- */
-export declare function insertTags(pageId: number, meta: PageData['meta'], trx: Knex.Transaction): Promise<void>;

package/lib/archive/db-ops/pages/write/insert-tags.js

@@ -1,34 +0,0 @@
-import { eachSplitted } from '../../../../utils/array/each-splitted.js';
-import { extractTagsForArchive } from '../../../meta/extract-tags-for-archive.js';
-/**
- * Replaces the page's Wappalyzer tag rows with the freshly captured set.
- * Called inside `updatePage`'s transaction unconditionally — tag
- * detection draws on `<script src>` / `<iframe src>` / window globals /
- * response headers, not the HTML body, so external pages that skip
- * rendering still contribute tags.
- *
- * Same empty-guard as `insertJsonLd`: an empty array does not wipe
- * prior rows on a degraded re-scrape.
- * @param pageId
- * @param meta
- * @param trx
- */
-export async function insertTags(pageId, meta, trx) {
-    const partial = extractTagsForArchive(meta.tags);
-    if (partial.length === 0)
-        return;
-    const rows = partial.map((p) => ({
-        pageId,
-        provider: p.provider,
-        category: p.category,
-        externalId: p.externalId,
-        version: p.version,
-        confidence: p.confidence,
-        categories: JSON.stringify(p.categories),
-        sources: JSON.stringify(p.sources),
-    }));
-    await trx('page_tags').where('pageId', pageId).delete();
-    await eachSplitted(rows, 100, async (chunk) => {
-        await trx('page_tags').insert(chunk);
-    });
-}

package/lib/archive/meta/summarize-tags.d.ts

@@ -1,16 +0,0 @@
-import type { TagRow, TagsSummary } from './types.js';
-/**
- * Builds the {@link TagsSummary} object returned by `get-page-detail` from a
- * page's tag rows.
- *
- * Provides a `provider → unique IDs[]` map so consumers can answer questions
- * like "what GTM containers are on this page?" without fetching the full
- * `categories` / `sources` JSON columns.
- *
- * IDs within each provider are sorted and de-duplicated. Providers with no
- * `externalId` rows still appear in the map with an empty `[]` so the
- * caller knows the provider was detected.
- * @param rows - All `page_tags` rows for one page.
- * @returns Summary with `count` and `providerIds` map.
- */
-export declare function summarizeTags(rows: readonly TagRow[]): TagsSummary;

package/lib/archive/meta/summarize-tags.js

@@ -1,33 +0,0 @@
-/**
- * Builds the {@link TagsSummary} object returned by `get-page-detail` from a
- * page's tag rows.
- *
- * Provides a `provider → unique IDs[]` map so consumers can answer questions
- * like "what GTM containers are on this page?" without fetching the full
- * `categories` / `sources` JSON columns.
- *
- * IDs within each provider are sorted and de-duplicated. Providers with no
- * `externalId` rows still appear in the map with an empty `[]` so the
- * caller knows the provider was detected.
- * @param rows - All `page_tags` rows for one page.
- * @returns Summary with `count` and `providerIds` map.
- */
-export function summarizeTags(rows) {
-    const providerIds = {};
-    for (const row of rows) {
-        if (!(row.provider in providerIds)) {
-            providerIds[row.provider] = new Set();
-        }
-        if (row.externalId !== null) {
-            providerIds[row.provider].add(row.externalId);
-        }
-    }
-    const sorted = {};
-    for (const provider of Object.keys(providerIds).toSorted()) {
-        sorted[provider] = [...providerIds[provider]].toSorted();
-    }
-    return {
-        count: rows.length,
-        providerIds: sorted,
-    };
-}