@nitpicker/crawler 0.11.0 → 0.12.0

package/lib/archive/archive.d.ts

@@ -1,4 +1,4 @@
-import type { Config, PageSource } from './types.js';
+import type { Config, InventoryRunMeta, PageSource } from './types.js';
 import type { PageData, CrawlerError, Resource } from '../utils/types/types.js';
 import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
 import { ArchiveAccessor } from './archive-accessor.js';
@@ -84,6 +84,16 @@ export default class Archive extends ArchiveAccessor {
      * @returns URLs already present in `resources`.
      */
     getExistingResourceUrls(urls: readonly string[]): Promise<string[]>;
+    /**
+     * Look up the `source` column of a single page row by its URL key. Thin
+     * facade over {@link Database.getPageSourceByUrl} — exposes the lookup
+     * to the orchestrator so it can inject a `PageSourceLookup` into the
+     * Crawler for sub-resource lineage propagation on `--resume` /
+     * `--retry-failed` sessions.
+     * @param url - URL key in `url.withoutHashAndAuth` form.
+     * @returns The recorded `source`, or `undefined` when no row exists.
+     */
+    getPageSourceByUrl(url: string): Promise<PageSource | undefined>;
     /**
      * Retrieves a single recorded sub-resource by its URL.
      * @param urls - URL candidates to match against the stored resource URL.
@@ -103,6 +113,61 @@ export default class Archive extends ArchiveAccessor {
      * @returns The base URL string.
      */
     getUrl(): Promise<any>;
+    /**
+     * Pre-insert inventory non-HTML URLs as `source='inventory-seed'`
+     * placeholders in the `resources` table — the non-HTML counterpart of
+     * {@link Archive.insertInventorySeeds}. Replaces the previous per-URL
+     * `setResources` loop in `CrawlerOrchestrator.inventory` so the
+     * ingestion phase commits all non-HTML rows in one chunked round-trip
+     * per 500 (a 50k-URL inventory list dropped from minutes-inside-`.bak`
+     * to seconds).
+     *
+     * Thin facade over {@link Database.insertInventoryResources}.
+     * `ExURL.href` is the storage key for `resources.url` (matches what
+     * `insertResource` writes for the per-URL path); we normalise here so
+     * the orchestrator stays decoupled from the storage form.
+     * @param urls - Non-HTML inventory URLs to record. No-op when empty.
+     */
+    insertInventoryResources(urls: readonly ExURL[]): Promise<void>;
+    /**
+     * Pre-insert inventory HTML seeds as `scraped=0`, `source='inventory-seed'`
+     * placeholder pages so the URL is durably tracked in the archive **before**
+     * the scrape phase starts. Thin facade over
+     * {@link Database.insertInventorySeeds} — see that method's JSDoc for the
+     * Ctrl+C-tolerance rationale and the `getCrawlingState` interaction.
+     *
+     * `ExURL` inputs are normalised to `withoutHashAndAuth` here so the storage
+     * key matches what `#getIdByUrl` writes for crawled rows, keeping the
+     * crawled-wins downgrade and the existing-URL filter (`getExistingPageUrls`)
+     * lookups consistent.
+     * @param urls - HTML seed URLs to pre-insert. No-op when empty.
+     */
+    insertInventorySeeds(urls: readonly ExURL[]): Promise<void>;
+    /**
+     * Hostnames whose `crawl_errors` history is consistently DNS failures and
+     * for which no recent 2xx/3xx page or resource is recorded. Consumed by
+     * `CrawlerOrchestrator.#preloadDnsBurnedHostCache` to seed the DNS-burned
+     * host cache at re-open (append / inventory / retryFailed / resume), so
+     * the next crawl skips HEAD pre-flight on hosts the previous crawl
+     * already proved unreachable.
+     *
+     * Deliberately exposed only on `Archive` (writer-side) — read-only
+     * `ArchiveAccessor` (stub viewer) does not see this method so the
+     * stub's no-migration contract is preserved.
+     * @returns Lower-cased hostnames safe to short-circuit.
+     */
+    listDnsBurnedHostCandidates(): Promise<string[]>;
+    /**
+     * Appends one row to the `inventory_runs` audit log.
+     *
+     * Thin facade over {@link Database.recordInventoryRun} — keeps the
+     * orchestrator decoupled from the knex layer and gives a single
+     * write entry point that future Archive-level concerns (locking,
+     * mirror sync, etc.) can hook into without touching every caller.
+     * @param meta - The run metadata. Only `ran_at` is required.
+     * @returns The autoincremented `id` of the inserted row.
+     */
+    recordInventoryRun(meta: InventoryRunMeta): Promise<number>;
     /**
      * Releases the SQLite handle and the advisory lock **without** writing
      * the archive or removing `tmpDir`.
@@ -163,8 +228,13 @@ export default class Archive extends ArchiveAccessor {
      * destination edge is written, leaving the destination's stored title / meta /
      * anchors / images untouched.
      * @param pageInfo - The HEAD-resolved page data carrying the redirect chain.
+     * @param source - Inventory provenance for a brand-new destination row.
+     *   Forwarded to `recordRedirect` so the destination's `source` (and the
+     *   chain-intermediate `source` derived from it) lands on the inventory
+     *   label instead of the DB DEFAULT `'crawled'` when the orchestrator is
+     *   running an inventory pass. `undefined` keeps the DB DEFAULT.
      */
-    setRedirect(pageInfo: PageData): Promise<void>;
+    setRedirect(pageInfo: PageData, source?: PageSource): Promise<void>;
     /**
      * Stores a sub-resource (CSS, JS, image, etc.) in the archive database.
      * @param resource - The resource data to store.
@@ -234,6 +304,51 @@ export default class Archive extends ArchiveAccessor {
      * @returns An ArchiveAccessor instance for querying the archive data.
      */
     static connect(tmpDir: string, namespace?: string | null): Promise<ArchiveAccessor>;
+    /**
+     * Open a `.nitpicker` archive through the read-only tar cache.
+     *
+     * This is the fast path for read-only consumers (viewer, MCP, query
+     * CLI). It diverges from {@link Archive.open} in two important ways:
+     *
+     * 1. The extracted contents land in an OS-temp-scoped cache directory
+     *    keyed by the archive's `size + mtime_ns + ctime_ns` (see
+     *    {@link computeArchiveCacheKey}). Subsequent opens of the same
+     *    unchanged archive skip the untar entirely. A fresh 10 GB archive
+     *    pays the ~10 s untar cost once; reopens are instant.
+     * 2. The returned value is an {@link ArchiveAccessor} (read-only), not
+     *    an `Archive` (writer). Closing it tears down the DB handle but
+     *    leaves the cache directory in place for the next reader. The
+     *    OS's own temp-directory cleanup (macOS reboot, Linux
+     *    `systemd-tmpfiles`, Windows Disk Cleanup) reclaims stale
+     *    entries — we do not own eviction.
+     *
+     * Migrations: the writer-side migration stack
+     * (`initSchema` / `migrate*`) runs once at cache-miss extraction, so
+     * the cache directory always lands on the current schema before the
+     * read-only re-open. Cache hits then skip migrations entirely.
+     *
+     * Override the cache location with `NITPICKER_TAR_CACHE_DIR`. The
+     * disable switch (`NITPICKER_DISABLE_TAR_CACHE=1`) is honoured by
+     * the caller (`ArchiveManager.open` falls back to {@link Archive.open}
+     * in that case); this function itself always goes through the cache.
+     *
+     * Writer entry points (`crawl --append`, `crawl --retry-failed`) must
+     * NOT use this path — they need the lock + write-back semantics of
+     * {@link Archive.open}.
+     * @param filePath - Absolute path to the `.nitpicker` file.
+     * @param namespace - Optional namespace forwarded to {@link ArchiveAccessor}.
+     * @returns A read-only {@link ArchiveAccessor} backed by the cache directory.
+     * @example
+     * ```ts
+     * const accessor = await Archive.openCached('/path/to/site.nitpicker');
+     * try {
+     *   const summary = await getSummary(accessor);
+     * } finally {
+     *   await accessor.close(); // tears down DB handle, cacheDir persists.
+     * }
+     * ```
+     */
+    static openCached(filePath: string, namespace?: string | null): Promise<ArchiveAccessor>;
     /**
      * Creates a new archive at the specified file path.
      * Initializes a temporary working directory and a fresh SQLite database.

package/lib/archive/archive.js

@@ -1,6 +1,10 @@
 import path from 'node:path';
 import { ArchiveAccessor } from './archive-accessor.js';
 import { acquireArchiveLock } from './archive-lock.js';
+import { computeArchiveCacheKey } from './cache/compute-archive-cache-key.js';
+import { extractArchiveToCache } from './cache/extract-archive-to-cache.js';
+import { getArchiveCacheRoot } from './cache/get-archive-cache-root.js';
+import { resolveArchiveCacheDir } from './cache/resolve-archive-cache-dir.js';
 import { Database } from './database.js';
 import { dbLog, log, saveLog } from './debug.js';
 import { appendText } from './filesystem/append-text.js';
@@ -136,6 +140,18 @@ export default class Archive extends ArchiveAccessor {
     async getExistingResourceUrls(urls) {
         return this.#db.getExistingResourceUrls(urls);
     }
+    /**
+     * Look up the `source` column of a single page row by its URL key. Thin
+     * facade over {@link Database.getPageSourceByUrl} — exposes the lookup
+     * to the orchestrator so it can inject a `PageSourceLookup` into the
+     * Crawler for sub-resource lineage propagation on `--resume` /
+     * `--retry-failed` sessions.
+     * @param url - URL key in `url.withoutHashAndAuth` form.
+     * @returns The recorded `source`, or `undefined` when no row exists.
+     */
+    async getPageSourceByUrl(url) {
+        return this.#db.getPageSourceByUrl(url);
+    }
     /**
      * Retrieves a single recorded sub-resource by its URL.
      * @param urls - URL candidates to match against the stored resource URL.
@@ -161,6 +177,78 @@ export default class Archive extends ArchiveAccessor {
     async getUrl() {
         return this.#db.getBaseUrl();
     }
+    /**
+     * Pre-insert inventory non-HTML URLs as `source='inventory-seed'`
+     * placeholders in the `resources` table — the non-HTML counterpart of
+     * {@link Archive.insertInventorySeeds}. Replaces the previous per-URL
+     * `setResources` loop in `CrawlerOrchestrator.inventory` so the
+     * ingestion phase commits all non-HTML rows in one chunked round-trip
+     * per 500 (a 50k-URL inventory list dropped from minutes-inside-`.bak`
+     * to seconds).
+     *
+     * Thin facade over {@link Database.insertInventoryResources}.
+     * `ExURL.href` is the storage key for `resources.url` (matches what
+     * `insertResource` writes for the per-URL path); we normalise here so
+     * the orchestrator stays decoupled from the storage form.
+     * @param urls - Non-HTML inventory URLs to record. No-op when empty.
+     */
+    async insertInventoryResources(urls) {
+        if (urls.length === 0) {
+            return;
+        }
+        dbLog('Insert inventory resources: %d URL(s)', urls.length);
+        await this.#db.insertInventoryResources(urls.map((u) => u.href));
+    }
+    /**
+     * Pre-insert inventory HTML seeds as `scraped=0`, `source='inventory-seed'`
+     * placeholder pages so the URL is durably tracked in the archive **before**
+     * the scrape phase starts. Thin facade over
+     * {@link Database.insertInventorySeeds} — see that method's JSDoc for the
+     * Ctrl+C-tolerance rationale and the `getCrawlingState` interaction.
+     *
+     * `ExURL` inputs are normalised to `withoutHashAndAuth` here so the storage
+     * key matches what `#getIdByUrl` writes for crawled rows, keeping the
+     * crawled-wins downgrade and the existing-URL filter (`getExistingPageUrls`)
+     * lookups consistent.
+     * @param urls - HTML seed URLs to pre-insert. No-op when empty.
+     */
+    async insertInventorySeeds(urls) {
+        if (urls.length === 0) {
+            return;
+        }
+        dbLog('Insert inventory seeds: %d URL(s)', urls.length);
+        await this.#db.insertInventorySeeds(urls.map((u) => u.withoutHashAndAuth));
+    }
+    /**
+     * Hostnames whose `crawl_errors` history is consistently DNS failures and
+     * for which no recent 2xx/3xx page or resource is recorded. Consumed by
+     * `CrawlerOrchestrator.#preloadDnsBurnedHostCache` to seed the DNS-burned
+     * host cache at re-open (append / inventory / retryFailed / resume), so
+     * the next crawl skips HEAD pre-flight on hosts the previous crawl
+     * already proved unreachable.
+     *
+     * Deliberately exposed only on `Archive` (writer-side) — read-only
+     * `ArchiveAccessor` (stub viewer) does not see this method so the
+     * stub's no-migration contract is preserved.
+     * @returns Lower-cased hostnames safe to short-circuit.
+     */
+    async listDnsBurnedHostCandidates() {
+        return this.#db.listDnsBurnedHostCandidates();
+    }
+    /**
+     * Appends one row to the `inventory_runs` audit log.
+     *
+     * Thin facade over {@link Database.recordInventoryRun} — keeps the
+     * orchestrator decoupled from the knex layer and gives a single
+     * write entry point that future Archive-level concerns (locking,
+     * mirror sync, etc.) can hook into without touching every caller.
+     * @param meta - The run metadata. Only `ran_at` is required.
+     * @returns The autoincremented `id` of the inserted row.
+     */
+    async recordInventoryRun(meta) {
+        dbLog('Record inventory run: %s', meta.list_label ?? meta.ran_at);
+        return await this.#db.recordInventoryRun(meta);
+    }
     /**
      * Releases the SQLite handle and the advisory lock **without** writing
      * the archive or removing `tmpDir`.
@@ -242,10 +330,15 @@ export default class Archive extends ArchiveAccessor {
      * destination edge is written, leaving the destination's stored title / meta /
      * anchors / images untouched.
      * @param pageInfo - The HEAD-resolved page data carrying the redirect chain.
+     * @param source - Inventory provenance for a brand-new destination row.
+     *   Forwarded to `recordRedirect` so the destination's `source` (and the
+     *   chain-intermediate `source` derived from it) lands on the inventory
+     *   label instead of the DB DEFAULT `'crawled'` when the orchestrator is
+     *   running an inventory pass. `undefined` keeps the DB DEFAULT.
      */
-    async setRedirect(pageInfo) {
+    async setRedirect(pageInfo, source) {
         dbLog('Set redirect: %s', pageInfo.url.href);
-        await this.#db.recordRedirect(pageInfo);
+        await this.#db.recordRedirect(pageInfo, source);
     }
     /**
      * Stores a sub-resource (CSS, JS, image, etc.) in the archive database.
@@ -380,6 +473,58 @@ export default class Archive extends ArchiveAccessor {
         const archive = new ArchiveAccessor(tmpDir, db, namespace, { readOnly: true });
         return archive;
     }
+    /**
+     * Open a `.nitpicker` archive through the read-only tar cache.
+     *
+     * This is the fast path for read-only consumers (viewer, MCP, query
+     * CLI). It diverges from {@link Archive.open} in two important ways:
+     *
+     * 1. The extracted contents land in an OS-temp-scoped cache directory
+     *    keyed by the archive's `size + mtime_ns + ctime_ns` (see
+     *    {@link computeArchiveCacheKey}). Subsequent opens of the same
+     *    unchanged archive skip the untar entirely. A fresh 10 GB archive
+     *    pays the ~10 s untar cost once; reopens are instant.
+     * 2. The returned value is an {@link ArchiveAccessor} (read-only), not
+     *    an `Archive` (writer). Closing it tears down the DB handle but
+     *    leaves the cache directory in place for the next reader. The
+     *    OS's own temp-directory cleanup (macOS reboot, Linux
+     *    `systemd-tmpfiles`, Windows Disk Cleanup) reclaims stale
+     *    entries — we do not own eviction.
+     *
+     * Migrations: the writer-side migration stack
+     * (`initSchema` / `migrate*`) runs once at cache-miss extraction, so
+     * the cache directory always lands on the current schema before the
+     * read-only re-open. Cache hits then skip migrations entirely.
+     *
+     * Override the cache location with `NITPICKER_TAR_CACHE_DIR`. The
+     * disable switch (`NITPICKER_DISABLE_TAR_CACHE=1`) is honoured by
+     * the caller (`ArchiveManager.open` falls back to {@link Archive.open}
+     * in that case); this function itself always goes through the cache.
+     *
+     * Writer entry points (`crawl --append`, `crawl --retry-failed`) must
+     * NOT use this path — they need the lock + write-back semantics of
+     * {@link Archive.open}.
+     * @param filePath - Absolute path to the `.nitpicker` file.
+     * @param namespace - Optional namespace forwarded to {@link ArchiveAccessor}.
+     * @returns A read-only {@link ArchiveAccessor} backed by the cache directory.
+     * @example
+     * ```ts
+     * const accessor = await Archive.openCached('/path/to/site.nitpicker');
+     * try {
+     *   const summary = await getSummary(accessor);
+     * } finally {
+     *   await accessor.close(); // tears down DB handle, cacheDir persists.
+     * }
+     * ```
+     */
+    static async openCached(filePath, namespace = null) {
+        const cacheRoot = getArchiveCacheRoot();
+        const cacheKey = await computeArchiveCacheKey(filePath);
+        const cacheDir = resolveArchiveCacheDir(cacheRoot, cacheKey, filePath);
+        log('Open cached: %s (cacheDir=%s)', filePath, cacheDir);
+        await extractArchiveToCache(filePath, cacheRoot, cacheDir, cacheKey);
+        return await Archive.connect(cacheDir, namespace);
+    }
     /**
      * Creates a new archive at the specified file path.
      * Initializes a temporary working directory and a fresh SQLite database.

package/lib/archive/cache/compute-archive-cache-key.d.ts

@@ -0,0 +1,39 @@
+/**
+ * Derive a stable cache key from a `.nitpicker` file's inode metadata
+ * plus a head+tail content sample.
+ *
+ * Metadata fields:
+ *
+ * - `size` covers the bulk of accidental cache collision risk in O(1).
+ * - `mtime` (mod time) changes whenever the file's content is rewritten,
+ *   which is the common case (`crawl --append`, `crawl --retry-failed`,
+ *   `cp -f`, rsync).
+ * - `ctime` (inode change time) closes the `touch -m -t <past>` loophole:
+ *   even if a user resets `mtime` to fake "unchanged", the act of touching
+ *   bumps `ctime` on POSIX so the key still rolls.
+ *
+ * Head + tail digest:
+ *
+ * - On filesystems with low-resolution timestamps (FAT / exFAT / NFSv3 /
+ *   some Docker volume mounts where mtime/ctime are second-granular and
+ *   sometimes don't move on small appends), `size + mtime + ctime`
+ *   alone can stay identical across an in-place rewrite.
+ * - We hash the first {@link HEAD_TAIL_HASH_BYTES} bytes and the last
+ *   {@link HEAD_TAIL_HASH_BYTES} bytes to detect this. For tar archives
+ *   the head holds the first entry's header (which moves when the inner
+ *   directory's name changes) and the tail holds the last entry's data
+ *   (which always moves on `crawl --append` because the appended pages
+ *   land near the end of the tar stream).
+ * - Full-content sha256 was rejected: ~20-30 s on a 10 GB archive,
+ *   which is slower than the untar this cache is meant to avoid.
+ *
+ * The key is stable across symlinks (the caller resolves via
+ * `fs.realpath` upstream) but **not** across hardlinks pointing at a
+ * mutated inode — that is intentional, hardlinking is a power-user
+ * move and the user is expected to know the cache will share an entry.
+ * @param archivePath - Absolute path to the `.nitpicker` file.
+ * @returns A string of the form
+ *   `<size>-<mtime_ns>-<ctime_ns>-<headHex>-<tailHex>` suitable for use
+ *   as a directory-name component.
+ */
+export declare function computeArchiveCacheKey(archivePath: string): Promise<string>;

package/lib/archive/cache/compute-archive-cache-key.js

@@ -0,0 +1,95 @@
+import { createHash } from 'node:crypto';
+import { createReadStream } from 'node:fs';
+import fs from 'node:fs/promises';
+/**
+ * Number of bytes hashed from each end of the file to corner the case
+ * where two different archives share `size + mtime + ctime` by accident.
+ *
+ * 64 KiB is a sweet spot: on a 10 GB archive it reads 0.0006 % of the
+ * file (~1 ms on an SSD, ~5 ms on a spinning disk) while reliably
+ * sampling enough of both the tar leading header AND the tail (which
+ * for tar archives contains the last-written entry's data, so any real
+ * mutation moves at least one of the two windows).
+ */
+const HEAD_TAIL_HASH_BYTES = 64 * 1024;
+/**
+ * Cache-key segment substituted when a file is smaller than the
+ * head+tail sample window. In that case the head segment alone already
+ * spans the whole file (size column also rolls), so a separate tail
+ * digest is meaningless and we encode that explicitly rather than
+ * silently overlapping reads.
+ */
+const TAIL_NOT_NEEDED = 'short';
+/**
+ * Derive a stable cache key from a `.nitpicker` file's inode metadata
+ * plus a head+tail content sample.
+ *
+ * Metadata fields:
+ *
+ * - `size` covers the bulk of accidental cache collision risk in O(1).
+ * - `mtime` (mod time) changes whenever the file's content is rewritten,
+ *   which is the common case (`crawl --append`, `crawl --retry-failed`,
+ *   `cp -f`, rsync).
+ * - `ctime` (inode change time) closes the `touch -m -t <past>` loophole:
+ *   even if a user resets `mtime` to fake "unchanged", the act of touching
+ *   bumps `ctime` on POSIX so the key still rolls.
+ *
+ * Head + tail digest:
+ *
+ * - On filesystems with low-resolution timestamps (FAT / exFAT / NFSv3 /
+ *   some Docker volume mounts where mtime/ctime are second-granular and
+ *   sometimes don't move on small appends), `size + mtime + ctime`
+ *   alone can stay identical across an in-place rewrite.
+ * - We hash the first {@link HEAD_TAIL_HASH_BYTES} bytes and the last
+ *   {@link HEAD_TAIL_HASH_BYTES} bytes to detect this. For tar archives
+ *   the head holds the first entry's header (which moves when the inner
+ *   directory's name changes) and the tail holds the last entry's data
+ *   (which always moves on `crawl --append` because the appended pages
+ *   land near the end of the tar stream).
+ * - Full-content sha256 was rejected: ~20-30 s on a 10 GB archive,
+ *   which is slower than the untar this cache is meant to avoid.
+ *
+ * The key is stable across symlinks (the caller resolves via
+ * `fs.realpath` upstream) but **not** across hardlinks pointing at a
+ * mutated inode — that is intentional, hardlinking is a power-user
+ * move and the user is expected to know the cache will share an entry.
+ * @param archivePath - Absolute path to the `.nitpicker` file.
+ * @returns A string of the form
+ *   `<size>-<mtime_ns>-<ctime_ns>-<headHex>-<tailHex>` suitable for use
+ *   as a directory-name component.
+ */
+export async function computeArchiveCacheKey(archivePath) {
+    const stats = await fs.stat(archivePath, { bigint: true });
+    const fileSize = Number(stats.size);
+    const headHex = await sha256OfRange(archivePath, 0, Math.min(HEAD_TAIL_HASH_BYTES, fileSize));
+    const tailHex = fileSize > HEAD_TAIL_HASH_BYTES
+        ? await sha256OfRange(archivePath, fileSize - HEAD_TAIL_HASH_BYTES, HEAD_TAIL_HASH_BYTES)
+        : TAIL_NOT_NEEDED;
+    return `${stats.size}-${stats.mtimeNs}-${stats.ctimeNs}-${headHex}-${tailHex}`;
+}
+/**
+ * SHA-256 the requested byte range of a file via a single read stream.
+ * Truncated to 16 hex chars (64 bits) because the digest only needs to
+ * disambiguate within a `(size, mtime, ctime)` bucket — full 256-bit
+ * fingerprints would only waste filesystem path budget.
+ * @param filePath - Absolute path to the file.
+ * @param start - Inclusive byte offset to begin reading at.
+ * @param length - Number of bytes to read; 0 returns the empty-input digest.
+ * @returns A 16-character hex string.
+ */
+async function sha256OfRange(filePath, start, length) {
+    const hash = createHash('sha256');
+    if (length === 0) {
+        return hash.digest('hex').slice(0, 16);
+    }
+    await new Promise((resolve, reject) => {
+        const stream = createReadStream(filePath, {
+            start,
+            end: start + length - 1,
+        });
+        stream.on('data', (chunk) => hash.update(chunk));
+        stream.on('end', () => resolve());
+        stream.on('error', (error) => reject(error));
+    });
+    return hash.digest('hex').slice(0, 16);
+}

package/lib/archive/cache/extract-archive-to-cache.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Untar a `.nitpicker` archive into the given cache directory.
+ *
+ * Concurrency / re-entry contract:
+ *
+ * - Two viewers opening the same archive race for the cache lock; the
+ *   loser waits and recheck-loops until the winner writes the ready
+ *   marker, then short-circuits.
+ * - Same-process concurrent callers dedupe through
+ *   `inFlightByCacheDir` so they share one extraction promise.
+ *
+ * Per-archive staging:
+ *
+ * - The tar's inner directory is extracted into `${cacheDir}.staging/`
+ *   first, then atomically renamed into `cacheDir`. Crucially, the
+ *   staging path is keyed off `cacheDir` (not the tar's inner directory
+ *   name) so two archives that happen to share an inner-dir name never
+ *   collide on the same staging slot.
+ *
+ * Integrity guard:
+ *
+ * - After rename, we run the writer-side `Database.connect` so all
+ *   migrations apply (column adds, new tables) and the version check
+ *   fires before the ready marker is written. A corrupt or incompatible
+ *   archive therefore poisons no cache entry — the cacheDir is removed
+ *   and the next caller retries from scratch.
+ *
+ * TOCTOU guard:
+ *
+ * - The cache key is recomputed from the file's stat AFTER the untar
+ *   completes. If the file changed mid-flight (a concurrent
+ *   `crawl --append` rewrote it), the freshly-landed cache contents do
+ *   not correspond to the original key, so we abort and remove the
+ *   cacheDir so the caller upstream sees the inconsistency rather than
+ *   serving misattributed data.
+ * @param archivePath - Absolute path to the source `.nitpicker` file.
+ * @param cacheRoot - Absolute path to the cache root directory.
+ * @param cacheDir - Absolute path the extracted contents should end up at.
+ * @param cacheKey - The cache key used to derive `cacheDir`. Recomputed
+ *   after extraction to detect concurrent writers; must match.
+ * @returns Resolves once `cacheDir` is ready to be opened read-only.
+ */
+export declare function extractArchiveToCache(archivePath: string, cacheRoot: string, cacheDir: string, cacheKey: string): Promise<void>;