@nitpicker/crawler 0.11.0 → 0.13.0

package/README.md

@@ -1,12 +1,14 @@
 # @nitpicker/crawler
 
-ヘッドレスブラウザによる Web クローラーエンジン。
+ヘッドレスブラウザでWebサイトをクロールし、`.nitpicker` アーカイブを生成・更新する内部パッケージです。
 
-## 概要
+通常は [@nitpicker/cli](../cli/README.md) の `crawl` コマンドから利用します。
 
-Puppeteer を使用して Web サイトをクロールし、各ページのメタデータ・リンク構造・ネットワークリソース・レンダリング後 HTML スナップショットを SQLite ベースのアーカイブ（`.nitpicker`）に保存します。
+## 関連リンク
 
-このパッケージは [Nitpicker](../../README.md) モノレポの内部パッケージです。単体での利用は想定していません。
+- [Nitpicker README](../../../README.md)
+- [CLI crawl docs](../cli/docs/crawl.md)
+- [ARCHITECTURE.md](../../../ARCHITECTURE.md)
 
 ## ライセンス
 

package/lib/archive/archive-accessor.d.ts

@@ -39,7 +39,7 @@ export declare class ArchiveAccessor extends EventEmitter<DatabaseEvent> {
      * tmpDir), where touching the filesystem would race with — or destroy —
      * the live crawler's working state.
      *
-     * Subclasses that own the archive's lifecycle (notably {@link Archive})
+     * Subclasses that own the archive's lifecycle (notably `Archive`)
      * override this to add write/cleanup steps.
      *
      * **Idempotent and concurrent-safe**: the first invocation captures the
@@ -171,7 +171,7 @@ export declare class ArchiveAccessor extends EventEmitter<DatabaseEvent> {
      * Retrieves a flat list of all resource URLs stored in the archive.
      * @returns An array of resource URL strings.
      */
-    getResourceUrlList(): Promise<any[]>;
+    getResourceUrlList(): Promise<string[]>;
     /**
      * Retrieves the Wappalyzer tag entries for the given page, parsed back
      * from the `page_tags` table.

package/lib/archive/archive-accessor.js

@@ -30,7 +30,7 @@ export class ArchiveAccessor extends EventEmitter {
      * Promise tracking an in-progress (or completed) close. `null` means the
      * accessor is open and idle; a settled promise means we are closed (the
      * accessor stays "closed" even if `db.destroy()` rejected, because there
-     * is nothing safe to retry — see {@link close}).
+     * is nothing safe to retry — see {@link ArchiveAccessor.close}).
      */
     #closeOnce = null;
     /** The SQLite database instance for querying archived data. */
@@ -91,7 +91,7 @@ export class ArchiveAccessor extends EventEmitter {
      * tmpDir), where touching the filesystem would race with — or destroy —
      * the live crawler's working state.
      *
-     * Subclasses that own the archive's lifecycle (notably {@link Archive})
+     * Subclasses that own the archive's lifecycle (notably `Archive`)
      * override this to add write/cleanup steps.
      *
      * **Idempotent and concurrent-safe**: the first invocation captures the

package/lib/archive/archive-lock.d.ts

@@ -34,5 +34,12 @@ export declare class ArchiveLockError extends Error {
  * @param tmpDir - Absolute path to the archive's temporary working directory.
  * @returns A release function to be called when the work is done.
  * @throws {ArchiveLockError} When the lock cannot be acquired even after a stale-lock retry.
+ * @example
+ * const releaseLock = await acquireArchiveLock(tmpDir);
+ * try {
+ *   // ... exclusive work against tmpDir ...
+ * } finally {
+ *   await releaseLock();
+ * }
  */
 export declare function acquireArchiveLock(tmpDir: string): Promise<() => Promise<void>>;

package/lib/archive/archive-lock.js

@@ -42,6 +42,13 @@ export class ArchiveLockError extends Error {
  * @param tmpDir - Absolute path to the archive's temporary working directory.
  * @returns A release function to be called when the work is done.
  * @throws {ArchiveLockError} When the lock cannot be acquired even after a stale-lock retry.
+ * @example
+ * const releaseLock = await acquireArchiveLock(tmpDir);
+ * try {
+ *   // ... exclusive work against tmpDir ...
+ * } finally {
+ *   await releaseLock();
+ * }
  */
 export async function acquireArchiveLock(tmpDir) {
     const lockPath = `${tmpDir}.lock`;

package/lib/archive/archive.d.ts

@@ -1,4 +1,4 @@
-import type { Config, PageSource } from './types.js';
+import type { Config, InventoryRunMeta, PageSource } from './types.js';
 import type { PageData, CrawlerError, Resource } from '../utils/types/types.js';
 import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
 import { ArchiveAccessor } from './archive-accessor.js';
@@ -14,6 +14,15 @@ import { ArchiveAccessor } from './archive-accessor.js';
  * Use the static factory methods ({@link Archive.create}, {@link Archive.open},
  * {@link Archive.resume}, {@link Archive.connect}) to obtain instances.
  * The constructor is private.
+ * @example
+ * const archive = await Archive.create({ filePath: '/path/to/site.nitpicker' });
+ * try {
+ *   await archive.setConfig(config);
+ *   const pageId = await archive.setPage(pageData);
+ * } finally {
+ *   // Writes the `.nitpicker` tar (if absent), removes tmpDir, releases the lock.
+ *   await archive.close();
+ * }
  */
 export default class Archive extends ArchiveAccessor {
     #private;
@@ -84,6 +93,16 @@ export default class Archive extends ArchiveAccessor {
      * @returns URLs already present in `resources`.
      */
     getExistingResourceUrls(urls: readonly string[]): Promise<string[]>;
+    /**
+     * Look up the `source` column of a single page row by its URL key. Thin
+     * facade over {@link Database.getPageSourceByUrl} — exposes the lookup
+     * to the orchestrator so it can inject a `PageSourceLookup` into the
+     * Crawler for sub-resource lineage propagation on `--resume` /
+     * `--retry-failed` sessions.
+     * @param url - URL key in `url.withoutHashAndAuth` form.
+     * @returns The recorded `source`, or `undefined` when no row exists.
+     */
+    getPageSourceByUrl(url: string): Promise<PageSource | undefined>;
     /**
      * Retrieves a single recorded sub-resource by its URL.
      * @param urls - URL candidates to match against the stored resource URL.
@@ -102,7 +121,61 @@ export default class Archive extends ArchiveAccessor {
      * Retrieves the base URL of the crawl session from the archive database.
      * @returns The base URL string.
      */
-    getUrl(): Promise<any>;
+    getUrl(): Promise<string>;
+    /**
+     * Pre-insert inventory non-HTML URLs as `source='inventory-seed'`
+     * placeholders in the `resources` table — the non-HTML counterpart of
+     * {@link Archive.insertInventorySeeds}. Rows are committed in chunked
+     * bulk inserts (500 per round-trip) rather than per-URL awaits — a
+     * per-URL loop would keep a 50k-URL inventory list inside the `.bak`
+     * window for minutes instead of seconds.
+     *
+     * Thin facade over {@link Database.insertInventoryResources}.
+     * `ExURL.href` is the storage key for `resources.url` (matches what
+     * `insertResource` writes for the per-URL path); we normalise here so
+     * the orchestrator stays decoupled from the storage form.
+     * @param urls - Non-HTML inventory URLs to record. No-op when empty.
+     */
+    insertInventoryResources(urls: readonly ExURL[]): Promise<void>;
+    /**
+     * Pre-insert inventory HTML seeds as `scraped=0`, `source='inventory-seed'`
+     * placeholder pages so the URL is durably tracked in the archive **before**
+     * the scrape phase starts. Thin facade over
+     * {@link Database.insertInventorySeeds} — see that method's JSDoc for the
+     * Ctrl+C-tolerance rationale and the `getCrawlingState` interaction.
+     *
+     * `ExURL` inputs are normalised to `withoutHashAndAuth` here so the storage
+     * key matches what `resolveContentItemId` writes for crawled rows, keeping the
+     * crawled-wins downgrade and the existing-URL filter (`getExistingPageUrls`)
+     * lookups consistent.
+     * @param urls - HTML seed URLs to pre-insert. No-op when empty.
+     */
+    insertInventorySeeds(urls: readonly ExURL[]): Promise<void>;
+    /**
+     * Hostnames whose `crawl_errors` history is consistently DNS failures and
+     * for which no recent 2xx/3xx page or resource is recorded. Consumed by
+     * `CrawlerOrchestrator.#preloadDnsBurnedHostCache` to seed the DNS-burned
+     * host cache at re-open (append / inventory / retryFailed / resume), so
+     * the next crawl skips HEAD pre-flight on hosts the previous crawl
+     * already proved unreachable.
+     *
+     * Deliberately exposed only on `Archive` (writer-side) — read-only
+     * `ArchiveAccessor` (stub viewer) does not see this method so the
+     * stub's no-migration contract is preserved.
+     * @returns Lower-cased hostnames safe to short-circuit.
+     */
+    listDnsBurnedHostCandidates(): Promise<string[]>;
+    /**
+     * Appends one row to the `inventory_runs` audit log.
+     *
+     * Thin facade over {@link Database.recordInventoryRun} — keeps the
+     * orchestrator decoupled from the knex layer and gives a single
+     * write entry point that future Archive-level concerns (locking,
+     * mirror sync, etc.) can hook into without touching every caller.
+     * @param meta - The run metadata. Only `ran_at` is required.
+     * @returns The autoincremented `id` of the inserted row.
+     */
+    recordInventoryRun(meta: InventoryRunMeta): Promise<number>;
     /**
      * Releases the SQLite handle and the advisory lock **without** writing
      * the archive or removing `tmpDir`.
@@ -115,6 +188,24 @@ export default class Archive extends ArchiveAccessor {
      * are mutually exclusive (the first one called wins).
      */
     releaseHandle(): Promise<void>;
+    /**
+     * Replaces the archive's analysis violations with a fresh SQL-backed set.
+     *
+     * Thin facade over {@link Database.replaceAnalysisViolations}; kept on
+     * `Archive` so the analyze pipeline can persist violations without
+     * reaching into the low-level database class directly.
+     * @param violations - Flat analyze violations.
+     */
+    replaceAnalysisViolations(violations: readonly {
+        validator: string;
+        severity: string;
+        rule: string;
+        code?: string | null;
+        message: string;
+        url: string;
+        line?: number | null;
+        col?: number | null;
+    }[]): Promise<void>;
     /**
      * Promote previously-external pages that now fall under the (possibly extended)
      * scope back to a pending state so that the crawler re-scrapes them as fully
@@ -163,8 +254,13 @@ export default class Archive extends ArchiveAccessor {
      * destination edge is written, leaving the destination's stored title / meta /
      * anchors / images untouched.
      * @param pageInfo - The HEAD-resolved page data carrying the redirect chain.
+     * @param source - Inventory provenance for a brand-new destination row.
+     *   Forwarded to `recordRedirect` so the destination's `source` (and the
+     *   chain-intermediate `source` derived from it) lands on the inventory
+     *   label instead of the DB DEFAULT `'crawled'` when the orchestrator is
+     *   running an inventory pass. `undefined` keeps the DB DEFAULT.
      */
-    setRedirect(pageInfo: PageData): Promise<void>;
+    setRedirect(pageInfo: PageData, source?: PageSource): Promise<void>;
     /**
      * Stores a sub-resource (CSS, JS, image, etc.) in the archive database.
      * @param resource - The resource data to store.
@@ -217,23 +313,89 @@ export default class Archive extends ArchiveAccessor {
     /** The prefix used for temporary working directories during archive operations. */
     static TMP_DIR_PREFIX: string;
     /**
-     * Opens a read-only connection to an existing archive's database.
+     * Opens a connection to an existing archive's database, defaulting to
+     * read-only.
      *
-     * Returns an {@link ArchiveAccessor} that provides query methods
-     * without the ability to modify or write the archive. The DB is opened
-     * in **read-only mode**: no schema migrations run, and the connection
+     * Returns an {@link ArchiveAccessor} that provides query methods. In the
+     * default read-only mode, no schema migrations run and the connection
      * refuses to resurrect a missing parent directory or db file (so a
      * TOCTOU window between source classification and this call cannot
-     * silently produce an empty phantom tmpDir).
+     * silently produce an empty phantom tmpDir); the returned accessor is
+     * also marked read-only so consumer-facing helpers (e.g.
+     * {@link ArchiveAccessor.getHtmlOfPage}) avoid any filesystem mutation
+     * on the user's tmpDir.
      *
-     * The returned accessor is also marked read-only so consumer-facing
-     * helpers (e.g. {@link ArchiveAccessor.getHtmlOfPage}) avoid any
-     * filesystem mutation on the user's tmpDir.
+     * `options.readOnly: false` is a narrow escape hatch for opening a
+     * second, writable connection to a `tmpDir` that {@link Archive.openCached}
+     * already extracted (and migrated) into an OS-temp cache directory —
+     * never the caller's live/interrupted crawl tmpDir, which must stay
+     * read-only. A read-only open (`Archive.openCached`/`ArchiveManager.open`)
+     * must never take this path itself — blocking or writing during what
+     * must be a read-only open is forbidden (issue #177). This escape
+     * hatch has no current production caller; any future
+     * one is responsible for its own cross-process coordination (see
+     * `acquireArchiveLock`) — this method does not acquire any lock itself.
      * @param tmpDir - The path to the temporary directory containing the database.
      * @param namespace - An optional namespace for scoping data access within the archive.
+     * @param options - Connection options.
+     * @param options.readOnly - Defaults to `true`. Pass `false` to obtain a
+     *   writable accessor against an already-extracted cache directory.
      * @returns An ArchiveAccessor instance for querying the archive data.
-     */
-    static connect(tmpDir: string, namespace?: string | null): Promise<ArchiveAccessor>;
+     * @example
+     * // Default (read-only) — safe for stub mode and cache reads:
+     * const accessor = await Archive.connect(tmpDir);
+     * @example
+     * // Writable escape hatch — only against a tar-cache extraction:
+     * const writable = await Archive.connect(cacheDir, null, { readOnly: false });
+     */
+    static connect(tmpDir: string, namespace?: string | null, options?: {
+        readOnly?: boolean;
+    }): Promise<ArchiveAccessor>;
+    /**
+     * Open a `.nitpicker` archive through the read-only tar cache.
+     *
+     * This is the fast path for read-only consumers (viewer, MCP, query
+     * CLI). It diverges from {@link Archive.open} in two important ways:
+     *
+     * 1. The extracted contents land in an OS-temp-scoped cache directory
+     *    keyed by the archive's `size + mtime_ns + ctime_ns` (see
+     *    {@link computeArchiveCacheKey}). Subsequent opens of the same
+     *    unchanged archive skip the untar entirely. A fresh 10 GB archive
+     *    pays the ~10 s untar cost once; reopens are instant.
+     * 2. The returned value is an {@link ArchiveAccessor} (read-only), not
+     *    an `Archive` (writer). Closing it tears down the DB handle but
+     *    leaves the cache directory in place for the next reader. The
+     *    OS's own temp-directory cleanup (macOS reboot, Linux
+     *    `systemd-tmpfiles`, Windows Disk Cleanup) reclaims stale
+     *    entries — we do not own eviction.
+     *
+     * Migrations: the writer-side migration stack
+     * (`initSchema` / `migrate*`) runs once at cache-miss extraction, so
+     * the cache directory always lands on the current schema before the
+     * read-only re-open. Cache hits then skip migrations entirely.
+     *
+     * Override the cache location with `NITPICKER_TAR_CACHE_DIR`. The
+     * disable switch (`NITPICKER_DISABLE_TAR_CACHE=1`) is honoured by
+     * the caller (`ArchiveManager.open` falls back to {@link Archive.open}
+     * in that case); this function itself always goes through the cache.
+     *
+     * Writer entry points (`crawl --append`, `crawl --retry-failed`) must
+     * NOT use this path — they need the lock + write-back semantics of
+     * {@link Archive.open}.
+     * @param filePath - Absolute path to the `.nitpicker` file.
+     * @param namespace - Optional namespace forwarded to {@link ArchiveAccessor}.
+     * @returns A read-only {@link ArchiveAccessor} backed by the cache directory.
+     * @example
+     * ```ts
+     * const accessor = await Archive.openCached('/path/to/site.nitpicker');
+     * try {
+     *   const summary = await getSummary(accessor);
+     * } finally {
+     *   await accessor.close(); // tears down DB handle, cacheDir persists.
+     * }
+     * ```
+     */
+    static openCached(filePath: string, namespace?: string | null): Promise<ArchiveAccessor>;
     /**
      * Creates a new archive at the specified file path.
      * Initializes a temporary working directory and a fresh SQLite database.

package/lib/archive/archive.js

@@ -1,6 +1,10 @@
 import path from 'node:path';
 import { ArchiveAccessor } from './archive-accessor.js';
 import { acquireArchiveLock } from './archive-lock.js';
+import { computeArchiveCacheKey } from './cache/compute-archive-cache-key.js';
+import { extractArchiveToCache } from './cache/extract-archive-to-cache.js';
+import { getArchiveCacheRoot } from './cache/get-archive-cache-root.js';
+import { resolveArchiveCacheDir } from './cache/resolve-archive-cache-dir.js';
 import { Database } from './database.js';
 import { dbLog, log, saveLog } from './debug.js';
 import { appendText } from './filesystem/append-text.js';
@@ -23,6 +27,15 @@ import { untar } from './filesystem/untar.js';
  * Use the static factory methods ({@link Archive.create}, {@link Archive.open},
  * {@link Archive.resume}, {@link Archive.connect}) to obtain instances.
  * The constructor is private.
+ * @example
+ * const archive = await Archive.create({ filePath: '/path/to/site.nitpicker' });
+ * try {
+ *   await archive.setConfig(config);
+ *   const pageId = await archive.setPage(pageData);
+ * } finally {
+ *   // Writes the `.nitpicker` tar (if absent), removes tmpDir, releases the lock.
+ *   await archive.close();
+ * }
  */
 export default class Archive extends ArchiveAccessor {
     /**
@@ -136,6 +149,18 @@ export default class Archive extends ArchiveAccessor {
     async getExistingResourceUrls(urls) {
         return this.#db.getExistingResourceUrls(urls);
     }
+    /**
+     * Look up the `source` column of a single page row by its URL key. Thin
+     * facade over {@link Database.getPageSourceByUrl} — exposes the lookup
+     * to the orchestrator so it can inject a `PageSourceLookup` into the
+     * Crawler for sub-resource lineage propagation on `--resume` /
+     * `--retry-failed` sessions.
+     * @param url - URL key in `url.withoutHashAndAuth` form.
+     * @returns The recorded `source`, or `undefined` when no row exists.
+     */
+    async getPageSourceByUrl(url) {
+        return this.#db.getPageSourceByUrl(url);
+    }
     /**
      * Retrieves a single recorded sub-resource by its URL.
      * @param urls - URL candidates to match against the stored resource URL.
@@ -161,6 +186,77 @@ export default class Archive extends ArchiveAccessor {
     async getUrl() {
         return this.#db.getBaseUrl();
     }
+    /**
+     * Pre-insert inventory non-HTML URLs as `source='inventory-seed'`
+     * placeholders in the `resources` table — the non-HTML counterpart of
+     * {@link Archive.insertInventorySeeds}. Rows are committed in chunked
+     * bulk inserts (500 per round-trip) rather than per-URL awaits — a
+     * per-URL loop would keep a 50k-URL inventory list inside the `.bak`
+     * window for minutes instead of seconds.
+     *
+     * Thin facade over {@link Database.insertInventoryResources}.
+     * `ExURL.href` is the storage key for `resources.url` (matches what
+     * `insertResource` writes for the per-URL path); we normalise here so
+     * the orchestrator stays decoupled from the storage form.
+     * @param urls - Non-HTML inventory URLs to record. No-op when empty.
+     */
+    async insertInventoryResources(urls) {
+        if (urls.length === 0) {
+            return;
+        }
+        dbLog('Insert inventory resources: %d URL(s)', urls.length);
+        await this.#db.insertInventoryResources(urls.map((u) => u.href));
+    }
+    /**
+     * Pre-insert inventory HTML seeds as `scraped=0`, `source='inventory-seed'`
+     * placeholder pages so the URL is durably tracked in the archive **before**
+     * the scrape phase starts. Thin facade over
+     * {@link Database.insertInventorySeeds} — see that method's JSDoc for the
+     * Ctrl+C-tolerance rationale and the `getCrawlingState` interaction.
+     *
+     * `ExURL` inputs are normalised to `withoutHashAndAuth` here so the storage
+     * key matches what `resolveContentItemId` writes for crawled rows, keeping the
+     * crawled-wins downgrade and the existing-URL filter (`getExistingPageUrls`)
+     * lookups consistent.
+     * @param urls - HTML seed URLs to pre-insert. No-op when empty.
+     */
+    async insertInventorySeeds(urls) {
+        if (urls.length === 0) {
+            return;
+        }
+        dbLog('Insert inventory seeds: %d URL(s)', urls.length);
+        await this.#db.insertInventorySeeds(urls.map((u) => u.withoutHashAndAuth));
+    }
+    /**
+     * Hostnames whose `crawl_errors` history is consistently DNS failures and
+     * for which no recent 2xx/3xx page or resource is recorded. Consumed by
+     * `CrawlerOrchestrator.#preloadDnsBurnedHostCache` to seed the DNS-burned
+     * host cache at re-open (append / inventory / retryFailed / resume), so
+     * the next crawl skips HEAD pre-flight on hosts the previous crawl
+     * already proved unreachable.
+     *
+     * Deliberately exposed only on `Archive` (writer-side) — read-only
+     * `ArchiveAccessor` (stub viewer) does not see this method so the
+     * stub's no-migration contract is preserved.
+     * @returns Lower-cased hostnames safe to short-circuit.
+     */
+    async listDnsBurnedHostCandidates() {
+        return this.#db.listDnsBurnedHostCandidates();
+    }
+    /**
+     * Appends one row to the `inventory_runs` audit log.
+     *
+     * Thin facade over {@link Database.recordInventoryRun} — keeps the
+     * orchestrator decoupled from the knex layer and gives a single
+     * write entry point that future Archive-level concerns (locking,
+     * mirror sync, etc.) can hook into without touching every caller.
+     * @param meta - The run metadata. Only `ran_at` is required.
+     * @returns The autoincremented `id` of the inserted row.
+     */
+    async recordInventoryRun(meta) {
+        dbLog('Record inventory run: %s', meta.list_label ?? meta.ran_at);
+        return await this.#db.recordInventoryRun(meta);
+    }
     /**
      * Releases the SQLite handle and the advisory lock **without** writing
      * the archive or removing `tmpDir`.
@@ -179,6 +275,17 @@ export default class Archive extends ArchiveAccessor {
         this.#closeOnce = this.#runReleaseHandle();
         return this.#closeOnce;
     }
+    /**
+     * Replaces the archive's analysis violations with a fresh SQL-backed set.
+     *
+     * Thin facade over {@link Database.replaceAnalysisViolations}; kept on
+     * `Archive` so the analyze pipeline can persist violations without
+     * reaching into the low-level database class directly.
+     * @param violations - Flat analyze violations.
+     */
+    async replaceAnalysisViolations(violations) {
+        await this.#db.replaceAnalysisViolations(violations);
+    }
     /**
      * Promote previously-external pages that now fall under the (possibly extended)
      * scope back to a pending state so that the crawler re-scrapes them as fully
@@ -242,10 +349,15 @@ export default class Archive extends ArchiveAccessor {
      * destination edge is written, leaving the destination's stored title / meta /
      * anchors / images untouched.
      * @param pageInfo - The HEAD-resolved page data carrying the redirect chain.
+     * @param source - Inventory provenance for a brand-new destination row.
+     *   Forwarded to `recordRedirect` so the destination's `source` (and the
+     *   chain-intermediate `source` derived from it) lands on the inventory
+     *   label instead of the DB DEFAULT `'crawled'` when the orchestrator is
+     *   running an inventory pass. `undefined` keeps the DB DEFAULT.
      */
-    async setRedirect(pageInfo) {
+    async setRedirect(pageInfo, source) {
         dbLog('Set redirect: %s', pageInfo.url.href);
-        await this.#db.recordRedirect(pageInfo);
+        await this.#db.recordRedirect(pageInfo, source);
     }
     /**
      * Stores a sub-resource (CSS, JS, image, etc.) in the archive database.
@@ -359,27 +471,99 @@ export default class Archive extends ArchiveAccessor {
     /** The prefix used for temporary working directories during archive operations. */
     static TMP_DIR_PREFIX = '._nitpicker-';
     /**
-     * Opens a read-only connection to an existing archive's database.
+     * Opens a connection to an existing archive's database, defaulting to
+     * read-only.
      *
-     * Returns an {@link ArchiveAccessor} that provides query methods
-     * without the ability to modify or write the archive. The DB is opened
-     * in **read-only mode**: no schema migrations run, and the connection
+     * Returns an {@link ArchiveAccessor} that provides query methods. In the
+     * default read-only mode, no schema migrations run and the connection
      * refuses to resurrect a missing parent directory or db file (so a
      * TOCTOU window between source classification and this call cannot
-     * silently produce an empty phantom tmpDir).
+     * silently produce an empty phantom tmpDir); the returned accessor is
+     * also marked read-only so consumer-facing helpers (e.g.
+     * {@link ArchiveAccessor.getHtmlOfPage}) avoid any filesystem mutation
+     * on the user's tmpDir.
      *
-     * The returned accessor is also marked read-only so consumer-facing
-     * helpers (e.g. {@link ArchiveAccessor.getHtmlOfPage}) avoid any
-     * filesystem mutation on the user's tmpDir.
+     * `options.readOnly: false` is a narrow escape hatch for opening a
+     * second, writable connection to a `tmpDir` that {@link Archive.openCached}
+     * already extracted (and migrated) into an OS-temp cache directory —
+     * never the caller's live/interrupted crawl tmpDir, which must stay
+     * read-only. A read-only open (`Archive.openCached`/`ArchiveManager.open`)
+     * must never take this path itself — blocking or writing during what
+     * must be a read-only open is forbidden (issue #177). This escape
+     * hatch has no current production caller; any future
+     * one is responsible for its own cross-process coordination (see
+     * `acquireArchiveLock`) — this method does not acquire any lock itself.
      * @param tmpDir - The path to the temporary directory containing the database.
      * @param namespace - An optional namespace for scoping data access within the archive.
+     * @param options - Connection options.
+     * @param options.readOnly - Defaults to `true`. Pass `false` to obtain a
+     *   writable accessor against an already-extracted cache directory.
      * @returns An ArchiveAccessor instance for querying the archive data.
-     */
-    static async connect(tmpDir, namespace = null) {
-        const db = await Archive.#connectDB(tmpDir, { readOnly: true });
-        const archive = new ArchiveAccessor(tmpDir, db, namespace, { readOnly: true });
+     * @example
+     * // Default (read-only) — safe for stub mode and cache reads:
+     * const accessor = await Archive.connect(tmpDir);
+     * @example
+     * // Writable escape hatch — only against a tar-cache extraction:
+     * const writable = await Archive.connect(cacheDir, null, { readOnly: false });
+     */
+    static async connect(tmpDir, namespace = null, options = {}) {
+        const readOnly = options.readOnly ?? true;
+        const db = await Archive.#connectDB(tmpDir, { readOnly });
+        const archive = new ArchiveAccessor(tmpDir, db, namespace, { readOnly });
         return archive;
     }
+    /**
+     * Open a `.nitpicker` archive through the read-only tar cache.
+     *
+     * This is the fast path for read-only consumers (viewer, MCP, query
+     * CLI). It diverges from {@link Archive.open} in two important ways:
+     *
+     * 1. The extracted contents land in an OS-temp-scoped cache directory
+     *    keyed by the archive's `size + mtime_ns + ctime_ns` (see
+     *    {@link computeArchiveCacheKey}). Subsequent opens of the same
+     *    unchanged archive skip the untar entirely. A fresh 10 GB archive
+     *    pays the ~10 s untar cost once; reopens are instant.
+     * 2. The returned value is an {@link ArchiveAccessor} (read-only), not
+     *    an `Archive` (writer). Closing it tears down the DB handle but
+     *    leaves the cache directory in place for the next reader. The
+     *    OS's own temp-directory cleanup (macOS reboot, Linux
+     *    `systemd-tmpfiles`, Windows Disk Cleanup) reclaims stale
+     *    entries — we do not own eviction.
+     *
+     * Migrations: the writer-side migration stack
+     * (`initSchema` / `migrate*`) runs once at cache-miss extraction, so
+     * the cache directory always lands on the current schema before the
+     * read-only re-open. Cache hits then skip migrations entirely.
+     *
+     * Override the cache location with `NITPICKER_TAR_CACHE_DIR`. The
+     * disable switch (`NITPICKER_DISABLE_TAR_CACHE=1`) is honoured by
+     * the caller (`ArchiveManager.open` falls back to {@link Archive.open}
+     * in that case); this function itself always goes through the cache.
+     *
+     * Writer entry points (`crawl --append`, `crawl --retry-failed`) must
+     * NOT use this path — they need the lock + write-back semantics of
+     * {@link Archive.open}.
+     * @param filePath - Absolute path to the `.nitpicker` file.
+     * @param namespace - Optional namespace forwarded to {@link ArchiveAccessor}.
+     * @returns A read-only {@link ArchiveAccessor} backed by the cache directory.
+     * @example
+     * ```ts
+     * const accessor = await Archive.openCached('/path/to/site.nitpicker');
+     * try {
+     *   const summary = await getSummary(accessor);
+     * } finally {
+     *   await accessor.close(); // tears down DB handle, cacheDir persists.
+     * }
+     * ```
+     */
+    static async openCached(filePath, namespace = null) {
+        const cacheRoot = getArchiveCacheRoot();
+        const cacheKey = await computeArchiveCacheKey(filePath);
+        const cacheDir = resolveArchiveCacheDir(cacheRoot, cacheKey, filePath);
+        log('Open cached: %s (cacheDir=%s)', filePath, cacheDir);
+        await extractArchiveToCache(filePath, cacheRoot, cacheDir, cacheKey);
+        return await Archive.connect(cacheDir, namespace);
+    }
     /**
      * Creates a new archive at the specified file path.
      * Initializes a temporary working directory and a fresh SQLite database.

package/lib/archive/cache/compute-archive-cache-key.d.ts

@@ -0,0 +1,39 @@
+/**
+ * Derive a stable cache key from a `.nitpicker` file's inode metadata
+ * plus a head+tail content sample.
+ *
+ * Metadata fields:
+ *
+ * - `size` covers the bulk of accidental cache collision risk in O(1).
+ * - `mtime` (mod time) changes whenever the file's content is rewritten,
+ *   which is the common case (`crawl --append`, `crawl --retry-failed`,
+ *   `cp -f`, rsync).
+ * - `ctime` (inode change time) closes the `touch -m -t <past>` loophole:
+ *   even if a user resets `mtime` to fake "unchanged", the act of touching
+ *   bumps `ctime` on POSIX so the key still rolls.
+ *
+ * Head + tail digest:
+ *
+ * - On filesystems with low-resolution timestamps (FAT / exFAT / NFSv3 /
+ *   some Docker volume mounts where mtime/ctime are second-granular and
+ *   sometimes don't move on small appends), `size + mtime + ctime`
+ *   alone can stay identical across an in-place rewrite.
+ * - We hash the first {@link HEAD_TAIL_HASH_BYTES} bytes and the last
+ *   {@link HEAD_TAIL_HASH_BYTES} bytes to detect this. For tar archives
+ *   the head holds the first entry's header (which moves when the inner
+ *   directory's name changes) and the tail holds the last entry's data
+ *   (which always moves on `crawl --append` because the appended pages
+ *   land near the end of the tar stream).
+ * - Full-content sha256 was rejected: ~20-30 s on a 10 GB archive,
+ *   which is slower than the untar this cache is meant to avoid.
+ *
+ * The key is stable across symlinks (the caller resolves via
+ * `fs.realpath` upstream) but **not** across hardlinks pointing at a
+ * mutated inode — that is intentional, hardlinking is a power-user
+ * move and the user is expected to know the cache will share an entry.
+ * @param archivePath - Absolute path to the `.nitpicker` file.
+ * @returns A string of the form
+ *   `<size>-<mtime_ns>-<ctime_ns>-<headHex>-<tailHex>` suitable for use
+ *   as a directory-name component.
+ */
+export declare function computeArchiveCacheKey(archivePath: string): Promise<string>;