@nitpicker/crawler 0.9.0 → 0.12.0

package/lib/archive/archive.d.ts

@@ -1,12 +1,15 @@
-import type { Config } from './types.js';
+import type { Config, InventoryRunMeta, PageSource } from './types.js';
 import type { PageData, CrawlerError, Resource } from '../utils/types/types.js';
 import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
 import { ArchiveAccessor } from './archive-accessor.js';
 /**
- * Main archive class for creating, opening, resuming, and writing Nitpicker archive files (`.nitpicker`).
+ * Main archive class for creating, opening, resuming, and writing Nitpicker
+ * archive files (`.nitpicker`).
  *
- * An Archive wraps a SQLite database and optional HTML snapshots into a compressed
- * tar archive. It extends {@link ArchiveAccessor} to provide read access to stored data.
+ * An Archive wraps a single SQLite database into a tar archive. HTML
+ * bodies live inside the same DB as zstd-compressed BLOBs (see #75) — the
+ * tar payload is effectively just `db.sqlite`. It extends
+ * {@link ArchiveAccessor} to provide read access to stored data.
  *
  * Use the static factory methods ({@link Archive.create}, {@link Archive.open},
  * {@link Archive.resume}, {@link Archive.connect}) to obtain instances.
@@ -18,22 +21,42 @@ export default class Archive extends ArchiveAccessor {
      * The absolute file path of the archive (`.nitpicker` file).
      */
     get filePath(): string;
+    /**
+     * The intermediate directory `Archive.write()` produces by renaming
+     * `tmpDir` before tarring (`{cwd}/{archiveName}`). Exposed so the
+     * manager can include it in its cleanup-on-failure path: if `tar()`
+     * fails after the rename, this directory is orphaned and would
+     * otherwise be invisible to a `rmSync(tmpDir)` recovery.
+     */
+    get renamedDir(): string;
     private constructor();
     /**
      * @deprecated This method is no longer functional.
      */
     abort(): void;
     /**
-     * Appends an error entry to the archive's error log file.
+     * Records a crawler-level error to both the human-readable `error.log` (full
+     * stack, for debugging) and the structured `crawl_errors` table (queryable,
+     * for the `error-kinds` analysis). The cause is not classified here — it is
+     * derived on read. `error.log` keeps the full stack while `crawl_errors`
+     * stores `error.message`; both normally carry the same cause token (e.g.
+     * `ENOTFOUND`), so classification agrees across the two — only an error whose
+     * cause lives solely in deeper stack frames could differ.
      * @param error - The crawler error object containing process and URL information.
      */
     addError(error: CrawlerError): Promise<void>;
     /**
-     * Closes the archive. If the archive file does not yet exist on disk,
-     * it writes the archive first. If the temporary directory still exists,
-     * it is removed.
+     * Records a partial scrape failure against the page identified by `url`.
+     *
+     * The corresponding `pages` row is created on demand (or matched if it
+     * already exists), so the call works even if the page's normal data has
+     * not been written yet.
+     * @param url - URL of the affected page.
+     * @param phase - Scrape phase name (typically `'retryExhausted'`).
+     * @param message - Human-readable failure message.
+     * @param isExternal - Whether the URL is external. Defaults to `false`.
      */
-    close(): Promise<void>;
+    addPageError(url: string, phase: string, message: string, isExternal?: boolean): Promise<void>;
     /**
      * Retrieves the current crawling state, including lists of scraped and pending URLs.
      * @returns An object with `scraped` and `pending` URL arrays.
@@ -42,11 +65,121 @@ export default class Archive extends ArchiveAccessor {
         scraped: string[];
         pending: string[];
     }>;
+    /**
+     * Return the subset of `urls` that already exist as `pages.url`. Used by
+     * `CrawlerOrchestrator.inventory` to filter the user-supplied URL list
+     * down to "URLs that are NOT yet in the archive" — only those reach the
+     * HEAD / scrape pipeline. Existing URLs are skipped to keep the second
+     * (and N-th) `--inventory` pass non-destructive.
+     * @param urls - Candidate URLs in `withoutHashAndAuth` form.
+     * @returns URLs already present in `pages`.
+     */
+    getExistingPageUrls(urls: readonly string[]): Promise<string[]>;
+    /**
+     * Return the subset of `urls` that already exist as `resources.url`. See
+     * {@link Archive.getExistingPageUrls} — the resource-side counterpart used
+     * by inventory mode to skip URLs that are already tracked as
+     * sub-resources.
+     * @param urls - Candidate URLs.
+     * @returns URLs already present in `resources`.
+     */
+    getExistingResourceUrls(urls: readonly string[]): Promise<string[]>;
+    /**
+     * Look up the `source` column of a single page row by its URL key. Thin
+     * facade over {@link Database.getPageSourceByUrl} — exposes the lookup
+     * to the orchestrator so it can inject a `PageSourceLookup` into the
+     * Crawler for sub-resource lineage propagation on `--resume` /
+     * `--retry-failed` sessions.
+     * @param url - URL key in `url.withoutHashAndAuth` form.
+     * @returns The recorded `source`, or `undefined` when no row exists.
+     */
+    getPageSourceByUrl(url: string): Promise<PageSource | undefined>;
+    /**
+     * Retrieves a single recorded sub-resource by its URL.
+     * @param urls - URL candidates to match against the stored resource URL.
+     * @returns The raw resource row, or `null` if none match.
+     */
+    getResourceByUrl(urls: readonly string[]): Promise<import("./types.js").DB_Resource | null>;
+    /**
+     * Counts the number of pages already scraped as crawl targets in the archive.
+     *
+     * Lets the crawler initialize its session-progress counter on resume so the
+     * displayed HTML-page count accounts for previously-rendered pages.
+     * @returns The count of pages with `isTarget = 1` and `scraped = 1`.
+     */
+    getScrapedHtmlPageCount(): Promise<number>;
     /**
      * Retrieves the base URL of the crawl session from the archive database.
      * @returns The base URL string.
      */
     getUrl(): Promise<any>;
+    /**
+     * Pre-insert inventory non-HTML URLs as `source='inventory-seed'`
+     * placeholders in the `resources` table — the non-HTML counterpart of
+     * {@link Archive.insertInventorySeeds}. Replaces the previous per-URL
+     * `setResources` loop in `CrawlerOrchestrator.inventory` so the
+     * ingestion phase commits all non-HTML rows in one chunked round-trip
+     * per 500 (a 50k-URL inventory list dropped from minutes-inside-`.bak`
+     * to seconds).
+     *
+     * Thin facade over {@link Database.insertInventoryResources}.
+     * `ExURL.href` is the storage key for `resources.url` (matches what
+     * `insertResource` writes for the per-URL path); we normalise here so
+     * the orchestrator stays decoupled from the storage form.
+     * @param urls - Non-HTML inventory URLs to record. No-op when empty.
+     */
+    insertInventoryResources(urls: readonly ExURL[]): Promise<void>;
+    /**
+     * Pre-insert inventory HTML seeds as `scraped=0`, `source='inventory-seed'`
+     * placeholder pages so the URL is durably tracked in the archive **before**
+     * the scrape phase starts. Thin facade over
+     * {@link Database.insertInventorySeeds} — see that method's JSDoc for the
+     * Ctrl+C-tolerance rationale and the `getCrawlingState` interaction.
+     *
+     * `ExURL` inputs are normalised to `withoutHashAndAuth` here so the storage
+     * key matches what `#getIdByUrl` writes for crawled rows, keeping the
+     * crawled-wins downgrade and the existing-URL filter (`getExistingPageUrls`)
+     * lookups consistent.
+     * @param urls - HTML seed URLs to pre-insert. No-op when empty.
+     */
+    insertInventorySeeds(urls: readonly ExURL[]): Promise<void>;
+    /**
+     * Hostnames whose `crawl_errors` history is consistently DNS failures and
+     * for which no recent 2xx/3xx page or resource is recorded. Consumed by
+     * `CrawlerOrchestrator.#preloadDnsBurnedHostCache` to seed the DNS-burned
+     * host cache at re-open (append / inventory / retryFailed / resume), so
+     * the next crawl skips HEAD pre-flight on hosts the previous crawl
+     * already proved unreachable.
+     *
+     * Deliberately exposed only on `Archive` (writer-side) — read-only
+     * `ArchiveAccessor` (stub viewer) does not see this method so the
+     * stub's no-migration contract is preserved.
+     * @returns Lower-cased hostnames safe to short-circuit.
+     */
+    listDnsBurnedHostCandidates(): Promise<string[]>;
+    /**
+     * Appends one row to the `inventory_runs` audit log.
+     *
+     * Thin facade over {@link Database.recordInventoryRun} — keeps the
+     * orchestrator decoupled from the knex layer and gives a single
+     * write entry point that future Archive-level concerns (locking,
+     * mirror sync, etc.) can hook into without touching every caller.
+     * @param meta - The run metadata. Only `ran_at` is required.
+     * @returns The autoincremented `id` of the inserted row.
+     */
+    recordInventoryRun(meta: InventoryRunMeta): Promise<number>;
+    /**
+     * Releases the SQLite handle and the advisory lock **without** writing
+     * the archive or removing `tmpDir`.
+     *
+     * Use this when you need to detach from a freshly-created `Archive`
+     * without finalising it — fixtures producing a stub state for tests,
+     * tooling that wants to leave the tmpDir alive for `crawl --resume`,
+     * or any non-orchestrator caller that owns the lifecycle externally.
+     * Shares the same idempotency guard as {@link close}, so the two paths
+     * are mutually exclusive (the first one called wins).
+     */
+    releaseHandle(): Promise<void>;
     /**
      * Promote previously-external pages that now fall under the (possibly extended)
      * scope back to a pending state so that the crawler re-scrapes them as fully
@@ -56,30 +189,58 @@ export default class Archive extends ArchiveAccessor {
      * @returns The URLs that were repromoted.
      */
     repromoteExternalPages(scopes: ReadonlyMap<string, readonly ExURL[]>, options?: ParseURLOptions): Promise<string[]>;
+    /**
+     * Reset previously-failed pages back to pending so a follow-up crawl re-fetches them.
+     *
+     * Delegates to {@link Database.resetFailedPages}. See that method for the
+     * exact failure criteria (missing status / content type, or a 5xx status).
+     * @returns The URLs of the pages that were reset to pending.
+     */
+    resetFailedPages(): Promise<string[]>;
     /**
      * Stores the crawl configuration into the archive database.
      * @param config - The configuration object to store.
      */
     setConfig(config: Config): Promise<number[]>;
     /**
-     * Stores an external page's data in the archive database without saving a snapshot.
+     * Stores an external page's data in the archive database without storing
+     * an HTML snapshot. External-page rows carry only metadata (status, title,
+     * content-type), never a rendered body.
      * @param pageInfo - The page data to store.
+     * @param source - Provenance label for new rows. `undefined` leaves the DB DEFAULT (`'crawled'`).
      */
-    setExternalPage(pageInfo: PageData): Promise<void>;
+    setExternalPage(pageInfo: PageData, source?: PageSource): Promise<void>;
     /**
-     * Stores a crawled page's data in the archive database and optionally saves an HTML snapshot.
-     * If the snapshot file write fails, the HTML path in the database is cleared to prevent
-     * referencing a non-existent file, and the error is re-thrown.
+     * Stores a crawled page's data in the archive database, persisting the
+     * rendered HTML body as a zstd-compressed BLOB inside the same SQLite
+     * transaction. Storage is content-addressable: identical bodies across
+     * pages share a single `page_html_blobs` row.
      * @param pageInfo - The page data to store.
+     * @param source - Provenance label for new rows. `undefined` leaves the DB DEFAULT (`'crawled'`).
      * @returns The database ID of the stored page.
-     * @throws {Error} Re-throws any error from the snapshot file write after clearing the HTML path.
      */
-    setPage(pageInfo: PageData): Promise<number>;
+    setPage(pageInfo: PageData, source?: PageSource): Promise<number>;
+    /**
+     * Records a redirect edge without re-storing the destination's content.
+     *
+     * The crawler calls this (instead of {@link setPage}) when a URL redirects to
+     * a destination that has already been rendered (#73): only the source →
+     * destination edge is written, leaving the destination's stored title / meta /
+     * anchors / images untouched.
+     * @param pageInfo - The HEAD-resolved page data carrying the redirect chain.
+     * @param source - Inventory provenance for a brand-new destination row.
+     *   Forwarded to `recordRedirect` so the destination's `source` (and the
+     *   chain-intermediate `source` derived from it) lands on the inventory
+     *   label instead of the DB DEFAULT `'crawled'` when the orchestrator is
+     *   running an inventory pass. `undefined` keeps the DB DEFAULT.
+     */
+    setRedirect(pageInfo: PageData, source?: PageSource): Promise<void>;
     /**
      * Stores a sub-resource (CSS, JS, image, etc.) in the archive database.
      * @param resource - The resource data to store.
+     * @param source - Provenance label for new rows. `undefined` leaves the DB DEFAULT (`'crawled'`).
      */
-    setResources(resource: Resource): Promise<void>;
+    setResources(resource: Resource, source?: PageSource): Promise<void>;
     /**
      * Stores the referrer relationship between a resource and the page that references it.
      * @param params - An object containing `url` (the page URL) and `src` (the resource URL).
@@ -109,30 +270,85 @@ export default class Archive extends ArchiveAccessor {
      */
     updateConfig(patch: Partial<Config>): Promise<void>;
     /**
-     * Writes the archive to disk as a compressed `.nitpicker` file.
+     * Writes the archive to disk as a `.nitpicker` tar file.
      *
-     * This method compresses the HTML snapshot directory into a zip file,
-     * renames the temporary working directory, and creates the final tar archive.
-     * The temporary directory is removed after writing.
+     * Checkpoints the SQLite WAL so the database is self-contained inside
+     * `db.sqlite`, renames the temporary working directory to the archive's
+     * basename, and tars it into the final `.nitpicker`. The tar container
+     * holds a single `db.sqlite` file (the legacy `snapshot-html.zip` is gone
+     * — HTML lives as BLOBs in the DB), so finalisation is effectively a
+     * single-file copy with no per-snapshot syscalls.
      */
     write(): Promise<void>;
     /** The file extension for Nitpicker archive files (without the leading dot). */
     static FILE_EXTENSION: string;
-    /** The directory name used for storing HTML snapshots within the archive. */
-    static readonly SNAPSHOT_HTML_DIR = "snapshot-html";
     /** The filename of the SQLite database within the archive. */
     static readonly SQLITE_DB_FILE_NAME = "db.sqlite";
     /** The prefix used for temporary working directories during archive operations. */
     static TMP_DIR_PREFIX: string;
     /**
      * Opens a read-only connection to an existing archive's database.
+     *
      * Returns an {@link ArchiveAccessor} that provides query methods
-     * without the ability to modify or write the archive.
+     * without the ability to modify or write the archive. The DB is opened
+     * in **read-only mode**: no schema migrations run, and the connection
+     * refuses to resurrect a missing parent directory or db file (so a
+     * TOCTOU window between source classification and this call cannot
+     * silently produce an empty phantom tmpDir).
+     *
+     * The returned accessor is also marked read-only so consumer-facing
+     * helpers (e.g. {@link ArchiveAccessor.getHtmlOfPage}) avoid any
+     * filesystem mutation on the user's tmpDir.
      * @param tmpDir - The path to the temporary directory containing the database.
      * @param namespace - An optional namespace for scoping data access within the archive.
      * @returns An ArchiveAccessor instance for querying the archive data.
      */
     static connect(tmpDir: string, namespace?: string | null): Promise<ArchiveAccessor>;
+    /**
+     * Open a `.nitpicker` archive through the read-only tar cache.
+     *
+     * This is the fast path for read-only consumers (viewer, MCP, query
+     * CLI). It diverges from {@link Archive.open} in two important ways:
+     *
+     * 1. The extracted contents land in an OS-temp-scoped cache directory
+     *    keyed by the archive's `size + mtime_ns + ctime_ns` (see
+     *    {@link computeArchiveCacheKey}). Subsequent opens of the same
+     *    unchanged archive skip the untar entirely. A fresh 10 GB archive
+     *    pays the ~10 s untar cost once; reopens are instant.
+     * 2. The returned value is an {@link ArchiveAccessor} (read-only), not
+     *    an `Archive` (writer). Closing it tears down the DB handle but
+     *    leaves the cache directory in place for the next reader. The
+     *    OS's own temp-directory cleanup (macOS reboot, Linux
+     *    `systemd-tmpfiles`, Windows Disk Cleanup) reclaims stale
+     *    entries — we do not own eviction.
+     *
+     * Migrations: the writer-side migration stack
+     * (`initSchema` / `migrate*`) runs once at cache-miss extraction, so
+     * the cache directory always lands on the current schema before the
+     * read-only re-open. Cache hits then skip migrations entirely.
+     *
+     * Override the cache location with `NITPICKER_TAR_CACHE_DIR`. The
+     * disable switch (`NITPICKER_DISABLE_TAR_CACHE=1`) is honoured by
+     * the caller (`ArchiveManager.open` falls back to {@link Archive.open}
+     * in that case); this function itself always goes through the cache.
+     *
+     * Writer entry points (`crawl --append`, `crawl --retry-failed`) must
+     * NOT use this path — they need the lock + write-back semantics of
+     * {@link Archive.open}.
+     * @param filePath - Absolute path to the `.nitpicker` file.
+     * @param namespace - Optional namespace forwarded to {@link ArchiveAccessor}.
+     * @returns A read-only {@link ArchiveAccessor} backed by the cache directory.
+     * @example
+     * ```ts
+     * const accessor = await Archive.openCached('/path/to/site.nitpicker');
+     * try {
+     *   const summary = await getSummary(accessor);
+     * } finally {
+     *   await accessor.close(); // tears down DB handle, cacheDir persists.
+     * }
+     * ```
+     */
+    static openCached(filePath: string, namespace?: string | null): Promise<ArchiveAccessor>;
     /**
      * Creates a new archive at the specified file path.
      * Initializes a temporary working directory and a fresh SQLite database.
@@ -167,6 +383,28 @@ export default class Archive extends ArchiveAccessor {
      * @returns A formatted timestamp string.
      */
     static timestamp(): string;
+    /**
+     * Closes the archive. If the archive file does not yet exist on disk,
+     * it writes the archive first. If the temporary directory still exists,
+     * it is removed. The database connection is then closed via
+     * {@link ArchiveAccessor.close} (the base class owns the SQLite handle),
+     * and finally the archive's advisory lock is released.
+     *
+     * **Idempotent**: the first invocation captures the close promise;
+     * subsequent invocations (signal handlers, parallel teardowns, retried
+     * orchestrator paths) await the same promise instead of re-entering
+     * the destructive prologue on a half-mutated state. If the first
+     * close fails (e.g. ENOSPC during tar), the rejection propagates to
+     * all awaiters and the archive stays latched closed — there is no
+     * safe way to retry `write()` once `tmpDir` has been renamed.
+     *
+     * **Read-only consumers must not reach this override.** Anything that
+     * obtains an archive view via {@link Archive.connect} receives an
+     * {@link ArchiveAccessor} (not an `Archive`), so `close()` resolves to
+     * the safe base implementation — no `write()`, no `remove()`, no lock
+     * release — leaving the tmpDir intact for the live crawler.
+     */
+    close(): Promise<void>;
     /**
      * Retrieves the crawl configuration stored in the archive database.
      * @returns The configuration object.