@nitpicker/crawler 0.11.0 → 0.13.0

package/lib/crawler-orchestrator.js

@@ -5,11 +5,16 @@ import { sortUrl } from '@d-zero/shared/sort-url';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
 import pkg from '../package.json' with { type: 'json' };
 import Archive from './archive/archive.js';
+import { REQUIRED_FORMAT_VERSION } from './archive/meta/assert-compatible-version.js';
 import { clearDestinationCache } from './crawler/clear-destination-cache.js';
+import { clearDnsBurnedHostCache } from './crawler/clear-dns-burned-host-cache.js';
 import Crawler from './crawler/crawler.js';
-import { fetchDestination } from './crawler/fetch-destination.js';
+import { dnsBurnedHostCache } from './crawler/dns-burned-host-cache.js';
+import { dnsBurnedHostShortCircuitCounter } from './crawler/dns-burned-host-short-circuit-counter.js';
 import { findScopeEntry } from './crawler/find-scope-entry.js';
-import { isHtmlContentType } from './crawler/is-html-content-type.js';
+import { isLikelyHtmlUrl } from './crawler/is-likely-html-url.js';
+import { PreloadShortCircuitError } from './crawler/preload-short-circuit-error.js';
+import { protocolAgnosticKey } from './crawler/protocol-agnostic-key.js';
 import { crawlerLog, log } from './debug.js';
 import { normalizeToArray } from './normalize-to-array.js';
 import { resolveOutputPath } from './resolve-output-path.js';
@@ -20,6 +25,9 @@ import { WriteQueue } from './write-queue.js';
  * Default list of external URL prefixes excluded from crawling.
  * Includes social media sharing endpoints that are commonly linked
  * but provide no useful crawl data.
+ * @example
+ * // Merged ahead of user-supplied excludeUrls when a crawl starts:
+ * const excludeUrls = [...DEFAULT_EXCLUDED_EXTERNAL_URLS, 'https://ads.example.com'];
  */
 export const DEFAULT_EXCLUDED_EXTERNAL_URLS = [
     'https://social-plugins.line.me',
@@ -116,8 +124,16 @@ export class CrawlerOrchestrator extends EventEmitter {
                 const row = await this.#writeQueue.enqueue(() => this.#archive.getResourceByUrl(urls));
                 return row ? resourceRowToLookupResult(row) : null;
             },
-            // Inventory mode is opted into by `CrawlerOrchestrator.inventory`
-            // (see T3); the default crawl path stays in normal mode so new
+            // Let the crawler propagate the parent's source lineage to
+            // sub-resources on `--resume` / `--retry-failed` sessions, where
+            // `inventoryMode` is not in memory but the DB still remembers
+            // the parent's `source`. Without this, sub-resources captured
+            // during a re-render of an inventory-labelled page would fall
+            // back to the DB DEFAULT `'crawled'` and lose their
+            // `'inventory-discovered'` provenance.
+            lookupPageSource: async (url) => this.#archive.getPageSourceByUrl(url),
+            // Inventory mode is opted into by `CrawlerOrchestrator.inventory`;
+            // the default crawl path stays in normal mode so new
             // rows continue to land in pages/resources with the DB DEFAULT
             // `'crawled'` provenance label.
             inventoryMode: options?.inventoryMode ?? null,
@@ -139,22 +155,30 @@ export class CrawlerOrchestrator extends EventEmitter {
      * Sets up event listeners on the crawler, starts crawling, and resolves
      * when the crawl completes. Discovered pages, external pages, skipped pages,
      * and resources are forwarded to the archive for storage.
-     * @param list - The list of parsed URLs to crawl. The first URL is used as the root.
+     * @param list - The list of parsed URLs to crawl. May be empty when a resumed
+     *   session already has pending pages queued (for example `--retry-failed`).
      * @param opts - Optional crawl overrides.
      * @param opts.recursive - Whether discovered URLs are followed. Defaults to
      *   `!fromList` (recursive unless the archive was created from a URL list), so
      *   existing callers keep their behaviour; the retry flow passes it explicitly.
      * @returns A promise that resolves when crawling is complete.
-     * @throws {Error} If the URL list is empty.
      */
     async crawling(list, opts) {
-        const root = list[0];
-        if (!root) {
-            throw new Error('URL is empty');
-        }
         const writeQueue = this.#writeQueue;
         return new Promise((resolve, reject) => {
             this.#crawler.on('error', (error) => {
+                if (error.error instanceof PreloadShortCircuitError) {
+                    // DNS-burned host short-circuit: the underlying cause already
+                    // lives in `crawl_errors` from the original DNS failure.
+                    // Writing it again on every subsequent URL would amplify the
+                    // row count on each `--retry-failed` re-run and could even
+                    // inflate the preload selection on the next open. Drop it
+                    // here; `pages.status = -1` still gets set via the normal
+                    // scrape-error path (handleScrapeError → addPageError) so the
+                    // page record itself is unchanged.
+                    crawlerLog('Skipping addError for preload short-circuit: %s', error.url);
+                    return;
+                }
                 crawlerLog('On error: %O', error);
                 writeQueue
                     .enqueue(() => this.#archive.addError(error))
@@ -181,9 +205,9 @@ export class CrawlerOrchestrator extends EventEmitter {
                     .enqueue(() => this.#archive.addPageError(url, phase, message, isExternal))
                     .catch((error) => reject(error));
             });
-            this.#crawler.on('redirect', ({ result }) => {
+            this.#crawler.on('redirect', ({ result, source }) => {
                 writeQueue
-                    .enqueue(() => this.#archive.setRedirect(result))
+                    .enqueue(() => this.#archive.setRedirect(result, source))
                     .catch((error) => reject(error));
                 void this.emit('redirect', { result });
             });
@@ -238,6 +262,12 @@ export class CrawlerOrchestrator extends EventEmitter {
     /**
      * Write the archive to its configured file path.
      *
+     * The crawler's write path inserts directly into the 0.13 entity
+     * tables (`content_items` / `page_meta` / `anchor_edges` / …) during
+     * `crawling` / `append` / `resume` / `retryFailed` / `inventory`, so by
+     * the time `write()` is called those tables are already populated.
+     * This method just tars.
+     *
      * Emits `writeFileStart` before writing and `writeFileEnd` after
      * the write completes successfully.
      */
@@ -279,7 +309,14 @@ export class CrawlerOrchestrator extends EventEmitter {
         // Each positional URL is both a starting point and a scope entry.
         const rootHrefs = list.map((u) => u.withoutHash);
         await archive.setConfig({
-            version: pkg.version,
+            // `version` is the archive-format version (see
+            // `assertCompatibleVersion`), NOT the npm package version. Decoupled
+            // because format-breaking changes and code-release cadence are
+            // different concerns — a patch release must not silently bump the
+            // format version and reject older archives, and a dev build of an
+            // unreleased breaking change must be able to produce archives the
+            // same build can read back.
+            version: REQUIRED_FORMAT_VERSION,
             name: fileName,
             baseUrl: rootHrefs[0],
             roots: rootHrefs,
@@ -314,7 +351,7 @@ export class CrawlerOrchestrator extends EventEmitter {
         log('Config %O', config);
         await orchestrator.crawling(list);
         log('Crawling completed');
-        clearDestinationCache();
+        CrawlerOrchestrator.#finalizeCrawlSession();
         log('Set order natural URL sort');
         await archive.setUrlOrder();
         log('Sorting done');
@@ -400,8 +437,9 @@ export class CrawlerOrchestrator extends EventEmitter {
                 log('Archive %s', absFilePath);
                 log('New roots %O', newRoots);
                 log('Merged roots %O', mergedRoots);
+                await CrawlerOrchestrator.#preloadDnsBurnedHostCache(archive);
                 await orchestrator.crawling(newParsed);
-                clearDestinationCache();
+                CrawlerOrchestrator.#finalizeCrawlSession();
                 await archive.setUrlOrder();
                 await ignoreEnoent(unlinkFile(backupPath));
                 return orchestrator;
@@ -466,10 +504,20 @@ export class CrawlerOrchestrator extends EventEmitter {
      * @param inventoryUrls - Pre-read URL list (one URL per element).
      * @param options - Optional config overrides — most callers leave this blank and let the archived config flow through.
      * @param initializedCallback - Hook invoked once the orchestrator is constructed but before `crawling` runs (the CLI uses it to attach progress reporting).
+     * @param sourceFileSha256 - **Pre-computed** SHA-256 hex digest of the
+     *   source URL list. The orchestrator deliberately does NOT receive
+     *   the file path: the path is privacy-sensitive (leaks user-home /
+     *   OS structure when archives are shared) and we want it lifted off
+     *   this boundary so no future log line / breadcrumb / error message
+     *   inside the orchestrator can accidentally re-leak it. The CLI
+     *   computes the digest via `computeFileSha256(resolvedListFile)`
+     *   and passes it through here. Pass `null` for programmatic
+     *   callers that built `inventoryUrls` in-memory; the audit row's
+     *   `source_file_sha256` column will be `NULL`.
      * @returns The orchestrator instance after a successful inventory pass.
      * @throws {Error} When `inventoryUrls` is empty, the archive is in list mode, or pending URLs from a previous crawl remain unresolved.
      */
-    static async inventory(archivePath, inventoryUrls, options, initializedCallback) {
+    static async inventory(archivePath, inventoryUrls, options, initializedCallback, sourceFileSha256 = null) {
         if (inventoryUrls.length === 0) {
             throw new Error('inventory: URL list is empty');
         }
@@ -483,9 +531,21 @@ export class CrawlerOrchestrator extends EventEmitter {
             if (archived.fromList) {
                 throw new Error('Cannot run inventory on a list-mode archive: this archive was created with --list/--list-file and contains metadata-only pages. Create a fresh archive instead.');
             }
-            const { scraped, pending } = await archive.getCrawlingState();
+            const { pending } = await archive.getCrawlingState();
             if (pending.length > 0) {
-                throw new Error(`inventory: archive has ${pending.length} pending URLs from a previous crawl. Resume or retry-failed first so inventory does not mislabel them as 'inventory-discovered'.`);
+                // `getCrawlingState` returns the STRICT pending set — in-scope,
+                // anchor-referenced, `scraped=0` rows. Predicted-discard leaks
+                // and external anomalies are filtered out at the reader, so a
+                // non-empty pending here means the previous session genuinely
+                // stopped with interrupted in-scope work. A hard rejection is
+                // still not warranted (with a looser reader it would block
+                // legitimate inventory runs whenever leak rows polluted the
+                // count), so a warning is
+                // enough — the inventory pass continues and the crawled-wins
+                // source priority keeps stale labels stable even if some of
+                // the strict-pending rows happen to land on inventory seeds.
+                // eslint-disable-next-line no-console -- operator-facing warning, must be visible regardless of DEBUG filters
+                console.warn(`inventory: archive has ${pending.length} pending URLs from a previous crawl. Proceeding — crawled-wins priority keeps their labels stable. Consider \`--resume\` first if you want the prior work finalized.`);
             }
             // Parse + scope-classify the candidate URLs. sortUrl drops
             // unparseable strings; findScopeEntry separates in-scope from
@@ -514,7 +574,7 @@ export class CrawlerOrchestrator extends EventEmitter {
             }
             // Drop URLs that are already represented in the archive (either
             // as pages or resources). Comparison key is `withoutHashAndAuth`
-            // to mirror what `#getIdByUrl` / `insertResource` actually store.
+            // to mirror what `resolveContentItemId` / `insertResource` actually store.
             // Two independent reads — Promise.all halves the wait on large
             // archives where each `WHERE url IN (?)` chunk costs real I/O.
             const candidateUrls = inScope.map((u) => u.withoutHashAndAuth);
@@ -546,56 +606,115 @@ export class CrawlerOrchestrator extends EventEmitter {
             }
             const backupPath = absFilePath + '.bak';
             await copyFile(absFilePath, backupPath);
+            // Ingestion (pre-insert + audit) is `.bak`-protected — a failure
+            // there restores the archive and the operator reruns. Once
+            // ingestion completes and the `.bak` is released, the scrape
+            // phase runs without `.bak` protection: a Ctrl+C / crash leaves
+            // the pre-inserted `inventory-seed` rows in `pages` so
+            // `crawl --resume` recovers them via the strict-pending set
+            // (see {@link Database.getCrawlingState}'s `OR p.source != 'crawled'`
+            // clause). This flag steers the catch below.
+            let ingestionComplete = false;
             try {
-                const headResults = await Promise.all(novelUrls.map(async (url) => {
-                    try {
-                        const head = await fetchDestination({
-                            url,
-                            isExternal: false,
-                            userAgent: archived.userAgent,
-                        });
-                        return { url, head, error: null };
+                // Classify novel URLs by URL-extension heuristic (no I/O).
+                // Source file lists come from `ls` on the doc-root, so the
+                // extension reflects the real file type — a HEAD pre-flight
+                // here would be pure wasted I/O. Edge cases:
+                //
+                // - `.html` returning 404 / 200: the normal crawler HEAD/GET
+                //   path absorbs this because every HTML-classified URL is
+                //   fed through the dealer and gets its real HEAD/GET there.
+                //
+                // - Extensionless API endpoints (e.g. `/api/foo`) that the
+                //   server returns as `text/html`: `isLikelyHtmlUrl` accepts
+                //   them as HTML so the dealer's render path runs — the
+                //   real content-type wins downstream.
+                //
+                // - `.aspx` / `.do` / `.jsp` / other server-handler
+                //   extensions that the heuristic does NOT recognise as
+                //   HTML: these are classified as non-HTML here, recorded
+                //   as `resources` rows with all-null metadata, and never
+                //   get a HEAD/GET probe. The accepted trade-off for
+                //   `--inventory`'s "list of static-looking server files"
+                //   contract; sites that mix server-handlers into the
+                //   inventory list will need a follow-up `--retry-failed`
+                //   pass (or a re-`--inventory` with the corrected list)
+                //   to populate metadata.
+                //
+                // non-HTML rows are recorded with null status/content-type
+                // which is sufficient for `listUnusedResources` (referrer
+                // count = 0) but means downstream consumers must treat
+                // null as "not probed" rather than "failed".
+                const rawHtmlSeeds = [];
+                const nonHtmlSeeds = [];
+                for (const url of novelUrls) {
+                    if (isLikelyHtmlUrl(url)) {
+                        rawHtmlSeeds.push(url);
                     }
-                    catch (headError) {
-                        const error = headError instanceof Error ? headError : new Error(String(headError));
-                        return { url, head: null, error };
+                    else {
+                        nonHtmlSeeds.push(url);
                     }
-                }));
+                }
+                // Dedup HTML seeds by `protocolAgnosticKey` so an inventory
+                // list that mixes `http://` and `https://` for the same
+                // origin does not produce two `pages` rows that the dealer
+                // later collapses to one — the loser would otherwise stay
+                // `scraped=0, source='inventory-seed'` forever and look like
+                // a real recovery candidate on `--resume`. `getExistingPageUrls`
+                // keys on the full URL (with protocol), so it cannot catch
+                // the cross-scheme duplicate; this is the dedup boundary.
+                const seenKeys = new Set();
                 const htmlSeeds = [];
-                for (const result of headResults) {
-                    const { url, head, error } = result;
-                    if (error !== null) {
-                        // HEAD failure is recorded as a crawl_errors row so
-                        // the URL is visible in `query error-kinds`, but does
-                        // NOT abort the whole inventory pass — other novel
-                        // URLs may still succeed.
-                        await archive.addError({
-                            pid: process.pid,
-                            isMainProcess: true,
-                            url: url.href,
-                            isExternal: false,
-                            error,
-                        });
+                for (const url of rawHtmlSeeds) {
+                    const key = protocolAgnosticKey(url.withoutHashAndAuth);
+                    if (seenKeys.has(key)) {
                         continue;
                     }
-                    if (head.contentType == null || isHtmlContentType(head.contentType)) {
-                        htmlSeeds.push(url);
-                    }
-                    else {
-                        await archive.setResources({
-                            url,
-                            isExternal: false,
-                            isError: false,
-                            status: head.status,
-                            statusText: head.statusText,
-                            contentType: head.contentType,
-                            contentLength: head.contentLength,
-                            compress: false,
-                            cdn: false,
-                            headers: head.responseHeaders ?? null,
-                        }, 'inventory-seed');
-                    }
+                    seenKeys.add(key);
+                    htmlSeeds.push(url);
                 }
+                // Bulk-record non-HTML novel URLs in `resources` as
+                // `source='inventory-seed'` placeholders. A
+                // per-URL `await setResources(...)` loop would spend minutes
+                // inside the `.bak`-protected window on large inventory
+                // lists; the chunked bulk path collapses N round-trips
+                // to N/500.
+                await archive.insertInventoryResources(nonHtmlSeeds);
+                // Pre-insert HTML seeds as `scraped = 0`,
+                // `source = 'inventory-seed'` placeholders *before* the
+                // scrape phase, so a Ctrl+C between here and `setPage`
+                // cannot lose the URL. The strict-pending set picks
+                // these rows up on the next `--resume` via the
+                // `OR p.source != 'crawled'` clause.
+                await archive.insertInventorySeeds(htmlSeeds);
+                log('[inventory] %d HTML seed(s), %d non-HTML resource(s) recorded', htmlSeeds.length, nonHtmlSeeds.length);
+                // Audit row is written *inside* the `.bak` window: a libsql
+                // hiccup or transient lock on the INSERT aborts the ingestion
+                // and the `.bak` restore wipes the pre-inserted seeds too,
+                // so "either the whole run took or none of it did" holds at
+                // the ingestion boundary. Audit failures are deliberately
+                // NOT swallowed — inside the `.bak` window a restore is
+                // safe and useful (see
+                // {@link CrawlerOrchestrator.#writeInventoryRunRow}).
+                await CrawlerOrchestrator.#writeInventoryRunRow(archive, {
+                    inventoryUrlsCount: inventoryUrls.length,
+                    htmlSeedsCount: htmlSeeds.length,
+                    nonHtmlCount: nonHtmlSeeds.length,
+                    outOfScope,
+                    sourceFileSha256,
+                });
+                // Ingestion's DB writes are now committed. From here on a
+                // throw must NOT trigger the `.bak` restore (it would wipe
+                // the durable seeds + audit row). Setting the flag *before*
+                // the `.bak` unlink covers the rare Windows / antivirus
+                // path where `unlinkFile` itself fails with EBUSY/EPERM —
+                // the `.bak` may leak on disk for the operator to delete
+                // manually, but the archive state stays intact.
+                ingestionComplete = true;
+                // Release `.bak` — ingestion succeeded. Beyond this point a
+                // throw is the scrape phase's problem; the archive stays
+                // intact and the operator runs `--resume` to recover.
+                await ignoreEnoent(unlinkFile(backupPath));
                 // Config sent to the user-facing `initializedCallback`
                 // (matches the rest of the orchestrator's public surface —
                 // no inventory bookkeeping leaks out).
@@ -615,22 +734,33 @@ export class CrawlerOrchestrator extends EventEmitter {
                 };
                 if (htmlSeeds.length > 0) {
                     const orchestrator = new CrawlerOrchestrator(archive, orchestratorOptions);
+                    // Re-read pending *after* the pre-insert so the strict-
+                    // pending set includes the freshly inserted
+                    // `inventory-seed` rows; feed that into `crawler.resume`
+                    // and start a seedless `crawling([])` — the same pattern
+                    // `retryFailed` uses to drive the dealer from the
+                    // pending set alone (see retryFailed's
+                    // `crawling([], { recursive })` invocation).
+                    const { scraped: scrapedAfter, pending: pendingAfter } = await archive.getCrawlingState();
                     const resources = await archive.getResourceUrlList();
-                    // Empty pending (we rejected non-empty above) but feed
-                    // every already-scraped URL into `seen` so the Crawler's
-                    // link enqueueing path drops links that hit a known
-                    // page without re-rendering it.
-                    orchestrator.#crawler.resume(pending, scraped, resources, 0);
+                    // Pre-existing rendered HTML page count seeds the
+                    // session-spanning `pagesScraped` counter so the progress
+                    // header reads `internalDone(cumulative pagesScraped)`
+                    // rather than session-only — matches the `append` /
+                    // `retryFailed` / `resume` paths and avoids users reading
+                    // the parenthesised number as "inner pages dropped to N".
+                    const pagesScrapedOffset = await archive.getScrapedHtmlPageCount();
+                    orchestrator.#crawler.resume(pendingAfter, scrapedAfter, resources, pagesScrapedOffset);
                     if (initializedCallback) {
                         await initializedCallback(orchestrator, baseConfig);
                     }
                     log('Start inventory');
                     log('Archive %s', absFilePath);
                     log('HTML seeds %O', htmlSeeds.map((u) => u.href));
-                    await orchestrator.crawling(htmlSeeds, { recursive: true });
-                    clearDestinationCache();
+                    await CrawlerOrchestrator.#preloadDnsBurnedHostCache(archive);
+                    await orchestrator.crawling([], { recursive: true });
+                    CrawlerOrchestrator.#finalizeCrawlSession();
                     await archive.setUrlOrder();
-                    await ignoreEnoent(unlinkFile(backupPath));
                     return orchestrator;
                 }
                 // Only non-HTML URLs were imported — nothing left to render,
@@ -640,10 +770,34 @@ export class CrawlerOrchestrator extends EventEmitter {
                     await initializedCallback(orchestrator, baseConfig);
                 }
                 await archive.setUrlOrder();
-                await ignoreEnoent(unlinkFile(backupPath));
                 return orchestrator;
             }
             catch (error) {
+                if (ingestionComplete) {
+                    // Scrape phase failed; the pre-inserted seeds + audit
+                    // row are durable inside `tmpDir/db.sqlite` but not yet
+                    // on disk as a `.nitpicker` tar. The outer catch below
+                    // runs `archive.close()`, which sees the original
+                    // (pre-inventory) `.nitpicker` already on disk and
+                    // would just `remove(tmpDir)` — silently wiping every
+                    // `inventory-seed` row and the audit row.
+                    //
+                    // Persist the ingested state ourselves before letting
+                    // the outer catch unwind, then re-throw so the operator
+                    // learns about the scrape failure (and can recover via
+                    // `crawl --resume <archive>`). `releaseHandle` shares
+                    // the orchestrator's `#closeOnce` guard, so the outer
+                    // catch's `close()` becomes a no-op for the destructive
+                    // step and only runs `releaseLock` cleanup.
+                    try {
+                        await archive.write();
+                        await archive.releaseHandle();
+                    }
+                    catch (persistError) {
+                        throw new AggregateError([error, persistError], 'inventory scrape phase failed AND persisting the ingested state to disk also failed. The archive may be in an inconsistent state — check tmpDir.');
+                    }
+                    throw error;
+                }
                 try {
                     await copyFile(backupPath, absFilePath);
                     await ignoreEnoent(unlinkFile(backupPath));
@@ -732,8 +886,9 @@ export class CrawlerOrchestrator extends EventEmitter {
                 if (initializedCallback) {
                     await initializedCallback(orchestrator, config);
                 }
-                await orchestrator.crawling(rootsParsed, { recursive: config.recursive });
-                clearDestinationCache();
+                await CrawlerOrchestrator.#preloadDnsBurnedHostCache(archive);
+                await orchestrator.crawling([], { recursive: config.recursive });
+                CrawlerOrchestrator.#finalizeCrawlSession();
                 await archive.setUrlOrder();
                 await ignoreEnoent(unlinkFile(backupPath));
                 return orchestrator;
@@ -793,9 +948,93 @@ export class CrawlerOrchestrator extends EventEmitter {
         log('Data %s', stubPath);
         log('URL %s', url.href);
         log('Config %O', config);
+        await CrawlerOrchestrator.#preloadDnsBurnedHostCache(archive);
         await orchestrator.crawling([url]);
+        CrawlerOrchestrator.#finalizeCrawlSession();
         return orchestrator;
     }
+    /**
+     * Seeds {@link dnsBurnedHostCache} from `crawl_errors` history at re-open
+     * (append / inventory / retryFailed / resume). Called after Archive.open
+     * succeeds and before crawling starts, so the first URL on a burned host
+     * already short-circuits — no retry budget is spent on a dead host that
+     * the previous crawl already proved was dead.
+     *
+     * Fresh `crawling()` skips this — there is no archive history to seed
+     * from. Within-session learning still kicks in via the `onGiveUp` mark.
+     * @param archive - The opened archive whose `crawl_errors` is read.
+     */
+    static async #preloadDnsBurnedHostCache(archive) {
+        const hosts = await archive.listDnsBurnedHostCandidates();
+        for (const host of hosts) {
+            dnsBurnedHostCache.set(host, 'dns');
+        }
+        if (hosts.length > 0) {
+            // eslint-disable-next-line no-console
+            console.error(`[preload] DNS-burned hosts: ${hosts.length} (will short-circuit subsequent URLs)`);
+        }
+    }
+    /**
+     * Persist one `inventory_runs` audit row inside the ingestion phase of a
+     * `--inventory` invocation, before the `.bak` is released. Lives as a
+     * static helper because the audit-row shape (timestamp stamping + label
+     * auto-gen + the privacy-driven path elision documented below) is a
+     * cohesive concern that benefits from staying outside the long
+     * `inventory()` body even though it has a single caller.
+     *
+     * `ran_at` is stamped now (ingestion-completion timestamp; the scrape
+     * phase that may follow is treated as separate). `list_label` is
+     * auto-generated from `ran_at` when the CLI did not pass one — there is
+     * no `--label` flag, so this is always the auto form.
+     * `source_file_sha256` arrives pre-computed via
+     * `aggregates.sourceFileSha256` (the CLI's `inventoryCrawl` ran
+     * `computeFileSha256` against the input txt before the orchestrator
+     * was even invoked). The orchestrator boundary deliberately never sees
+     * the absolute path — see {@link InventoryRunAggregates} for the
+     * privacy rationale.
+     *
+     * **Audit-write failures abort the ingestion phase.** Swallowing them
+     * would only be justified if the audit were the last write after the
+     * scrape (re-throwing there would wipe a completed crawl); inside the
+     * `.bak`-protected ingestion phase the
+     * trade-off flips. A failed audit row is restorable: the outer catch
+     * copies `.bak` back over the archive and the operator reruns the
+     * (short) ingestion from scratch. That keeps `inventory_runs` honest
+     * (no "ran but unrecorded" rows) at the cost of one rerun.
+     *
+     * Forward-compat: if an explicit `--label` flag is ever added, thread
+     * `labelOverride` through {@link inventory} into the `aggregates`
+     * shape so the auto-name can be overridden.
+     * @param archive - The opened archive to write the audit row into.
+     * @param aggregates - The counts captured during the inventory pass; see {@link InventoryRunAggregates}.
+     */
+    static async #writeInventoryRunRow(archive, aggregates) {
+        const ranAt = new Date().toISOString();
+        await archive.recordInventoryRun({
+            ran_at: ranAt,
+            list_label: `inventory-${ranAt}`,
+            source_file_sha256: aggregates.sourceFileSha256,
+            total_lines: aggregates.inventoryUrlsCount,
+            new_pages: aggregates.htmlSeedsCount,
+            new_resources: aggregates.nonHtmlCount,
+            scope_skipped: aggregates.outOfScope,
+        });
+    }
+    /**
+     * Tears down session-scoped crawler caches and prints a short-circuit
+     * summary if any URL fetches were skipped. Invoked at every
+     * crawl-session boundary (`crawling` / `append` / `inventory` /
+     * `retryFailed` / `resume`).
+     */
+    static #finalizeCrawlSession() {
+        const skipped = dnsBurnedHostShortCircuitCounter.count;
+        if (skipped > 0) {
+            // eslint-disable-next-line no-console
+            console.error(`[preload] Short-circuited ${skipped} URL(s) on DNS-burned hosts`);
+        }
+        clearDestinationCache();
+        clearDnsBurnedHostCache();
+    }
 }
 /**
  * Await a filesystem promise but silently swallow only `ENOENT` errors. Any

package/lib/crawler.d.ts

@@ -7,8 +7,6 @@
 export * from './utils/types/types.js';
 export { eachSplitted } from './utils/array/each-splitted.js';
 export { DOMEvaluationError } from './utils/error/dom-evaluation-error.js';
-export { ErrorEmitter } from './utils/error/error-emitter.js';
-export type { ErrorEvent } from './utils/error/error-emitter.js';
 export * from './utils/object/clean-object.js';
 export { globalLog as log } from './utils/debug.js';
 export { ArchiveAccessor } from './archive/archive-accessor.js';
@@ -17,6 +15,8 @@ export { default as Page } from './archive/page.js';
 export { default as ArchiveResource } from './archive/resource.js';
 export * from './archive/types.js';
 export { default as Archive } from './archive/archive.js';
+export { isArchiveCacheDisabled } from './archive/cache/is-archive-cache-disabled.js';
+export { acquireArchiveLock, ArchiveLockError } from './archive/archive-lock.js';
 export { peekArchiveLockHolder } from './archive/peek-archive-lock.js';
 export type { ArchiveLockHolder } from './archive/peek-archive-lock.js';
 export type { FlatPageMetaColumns, JsonLdRow, JsonLdRowForInsert, TagRow, TagRowForInsert, JsonLdSummary, TagsSummary, TagInventoryEntry, PageDenormalizedColumns, } from './archive/meta/types.js';
@@ -25,3 +25,16 @@ export { REQUIRED_FORMAT_VERSION } from './archive/meta/assert-compatible-versio
 export { DEFAULT_EXCLUDED_EXTERNAL_URLS, CrawlerOrchestrator, } from './crawler-orchestrator.js';
 export * from './types.js';
 export * from './crawler/types.js';
+export { classifyErrorKind } from './classify-error-kind.js';
+export { computeFileSha256 } from './utils/compute-file-sha256.js';
+export { populateEntityTables } from './archive/populate-entity-tables/populate-entities.js';
+export type { PageDomPathResolver } from './archive/populate-entity-tables/populate-image-items.js';
+export { populateRefTables } from './archive/populate-ref-tables/populate-refs.js';
+export { populateContentTypeRefs } from './archive/populate-ref-tables/populate-content-type-refs.js';
+export { populateUrlRefs } from './archive/populate-ref-tables/populate-url-refs.js';
+export { populateTextRefs } from './archive/populate-ref-tables/populate-text-refs.js';
+export { populateJsonRefs } from './archive/populate-ref-tables/populate-json-refs.js';
+export { populateBlobRefs } from './archive/populate-ref-tables/populate-blob-refs.js';
+export { populateHeaderTables } from './archive/populate-ref-tables/populate-header-tables.js';
+export { loadResponseHeadersBySetIds } from './archive/db-ops/_shared/load-response-headers-by-set-ids.js';
+export { decodeJsonRef } from './archive/db-ops/_shared/decode-json-ref.js';

package/lib/crawler.js

@@ -8,7 +8,6 @@
 export * from './utils/types/types.js';
 export { eachSplitted } from './utils/array/each-splitted.js';
 export { DOMEvaluationError } from './utils/error/dom-evaluation-error.js';
-export { ErrorEmitter } from './utils/error/error-emitter.js';
 export * from './utils/object/clean-object.js';
 export { globalLog as log } from './utils/debug.js';
 // Archive
@@ -17,6 +16,8 @@ export { default as Page } from './archive/page.js';
 export { default as ArchiveResource } from './archive/resource.js';
 export * from './archive/types.js';
 export { default as Archive } from './archive/archive.js';
+export { isArchiveCacheDisabled } from './archive/cache/is-archive-cache-disabled.js';
+export { acquireArchiveLock, ArchiveLockError } from './archive/archive-lock.js';
 export { peekArchiveLockHolder } from './archive/peek-archive-lock.js';
 export { IncompatibleArchiveError } from './archive/meta/types.js';
 export { REQUIRED_FORMAT_VERSION } from './archive/meta/assert-compatible-version.js';
@@ -24,3 +25,25 @@ export { REQUIRED_FORMAT_VERSION } from './archive/meta/assert-compatible-versio
 export { DEFAULT_EXCLUDED_EXTERNAL_URLS, CrawlerOrchestrator, } from './crawler-orchestrator.js';
 export * from './types.js';
 export * from './crawler/types.js';
+export { classifyErrorKind } from './classify-error-kind.js';
+export { computeFileSha256 } from './utils/compute-file-sha256.js';
+// 0.13 ref-table population (issue #191, epic #103). Exposed as the
+// public seam that the migration script (`scripts/migrate-to-0.13.mjs`)
+// drives against an already-connected archive.
+// The individual sub-steps are also exported so the migration script can
+// resume mid-way if the caller decides to split the transaction.
+export { populateEntityTables } from './archive/populate-entity-tables/populate-entities.js';
+export { populateRefTables } from './archive/populate-ref-tables/populate-refs.js';
+export { populateContentTypeRefs } from './archive/populate-ref-tables/populate-content-type-refs.js';
+export { populateUrlRefs } from './archive/populate-ref-tables/populate-url-refs.js';
+export { populateTextRefs } from './archive/populate-ref-tables/populate-text-refs.js';
+export { populateJsonRefs } from './archive/populate-ref-tables/populate-json-refs.js';
+export { populateBlobRefs } from './archive/populate-ref-tables/populate-blob-refs.js';
+export { populateHeaderTables } from './archive/populate-ref-tables/populate-header-tables.js';
+// 0.13 read-side reconstruction primitives. Exported so downstream
+// readers (`@nitpicker/query`'s page-detail view) reconstruct
+// `responseHeaders` / json_refs payloads with the exact same merge and
+// decode semantics as the crawler's own read paths — one implementation,
+// no cross-package drift.
+export { loadResponseHeadersBySetIds } from './archive/db-ops/_shared/load-response-headers-by-set-ids.js';
+export { decodeJsonRef } from './archive/db-ops/_shared/decode-json-ref.js';