@nitpicker/crawler 0.8.0 → 0.11.0

package/lib/archive/archive-accessor.d.ts

@@ -4,15 +4,13 @@ import type { ParseURLOptions } from '@d-zero/shared/parse-url';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
 import Page from './page.js';
 import Resource from './resource.js';
-/**
- * Provides read-only access to an archive's database and stored data files.
- *
- * This class is the base for the `Archive` class and is also returned
- * by `Archive.connect` for read-only access to an existing archive.
- * It supports querying pages, anchors, referrers, resources, and custom data.
- */
 export declare class ArchiveAccessor extends EventEmitter<DatabaseEvent> {
     #private;
+    /**
+     * Whether this accessor was opened in read-only mode (no filesystem
+     * mutation on `tmpDir`).
+     */
+    get readOnly(): boolean;
     /**
      * The absolute path to the temporary working directory used by this accessor.
      */
@@ -23,8 +21,47 @@ export declare class ArchiveAccessor extends EventEmitter<DatabaseEvent> {
      * @param db - The Database instance for querying the SQLite database.
      * @param namespace - An optional namespace for scoping custom data storage.
      *                    When null, `setData` is not available.
+     * @param options - Construction options.
+     * @param options.readOnly - When `true`, helpers must not mutate the
+     *   filesystem under `tmpDir` (used for live-crawl / stub-mode opens
+     *   where any write would race the crawler).
      */
-    constructor(tmpDir: string, db: Database, namespace?: string | null);
+    constructor(tmpDir: string, db: Database, namespace?: string | null, options?: {
+        readOnly?: boolean;
+    });
+    /**
+     * Closes the underlying database connection.
+     *
+     * This is the **read-only** close path: it releases the SQLite handle and
+     * does nothing else. The temporary working directory is left untouched and
+     * no `.nitpicker` archive is produced. This makes it safe to call from
+     * read-only consumers (e.g. the viewer attached to an in-progress crawl's
+     * tmpDir), where touching the filesystem would race with — or destroy —
+     * the live crawler's working state.
+     *
+     * Subclasses that own the archive's lifecycle (notably {@link Archive})
+     * override this to add write/cleanup steps.
+     *
+     * **Idempotent and concurrent-safe**: the first invocation captures the
+     * close promise; later invocations (from the same caller, a shutdown
+     * signal handler, or a parallel manager teardown) await the same
+     * promise and resolve together. If `db.destroy()` rejects, the
+     * rejection propagates to *all* awaiters and the accessor stays
+     * latched closed — a hung knex pool is not safe to "retry close".
+     *
+     * **Bounded**: when the optional `timeoutMs` (default {@link
+     * DEFAULT_CLOSE_TIMEOUT_MS}) elapses before `db.destroy()` settles, the
+     * call resolves with a warning. This prevents a viewer shutdown from
+     * being held for the underlying pool's 10-minute `acquireTimeoutMillis`
+     * when the live crawler holds the SQLite write lock.
+     * @param options - Close options.
+     * @param options.timeoutMs - Milliseconds to wait for `db.destroy()`
+     *   before giving up. Use `Infinity` to wait indefinitely (only
+     *   advisable in tests and batch jobs that own the DB exclusively).
+     */
+    close(options?: {
+        timeoutMs?: number;
+    }): Promise<void>;
     /**
      * Retrieves anchor (link) data for a specific page by its database ID.
      * @param pageId - The database ID of the page whose anchors to retrieve.
@@ -51,13 +88,42 @@ export declare class ArchiveAccessor extends EventEmitter<DatabaseEvent> {
      */
     getData(name: string, format?: 'txt' | 'html'): Promise<string>;
     /**
-     * Reads the HTML content of a page snapshot from the archive.
-     * Supports reading from both unzipped directories and zipped snapshot archives.
-     * @param filePath - The relative file path to the HTML snapshot, or null.
-     * @param openZipped - Whether to attempt unzipping the snapshot archive. Defaults to `true`.
-     * @returns The HTML content as a string, or null if the snapshot is not found or filePath is null.
+     * Reads the HTML snapshot of a page from the archive.
+     *
+     * HTML is stored as zstd-compressed BLOBs in `page_html_blobs` (keyed by
+     * SHA-256 of the raw bytes) with `page_html_ref` linking `page_id → hash`.
+     * The read is a straight join + decompress; writer- and read-only (stub)
+     * accessors take the same code path because nothing here touches the
+     * filesystem.
+     *
+     * Returns `null` when the page row exists but has no stored body — for
+     * example a redirect source, a non-HTML resource (PDF), a page that
+     * failed to render, or an external page (whose row is metadata-only).
+     * Distinguishing "no body stored" from "empty body" is preserved: an
+     * empty HTML string returns `''`, not `null`.
+     *
+     * Throws if the page's referenced blob is missing or the codec marker
+     * is unrecognised — both indicate an archive that was truncated or
+     * written by a future tool, neither of which we silently paper over.
+     * @param pageId - The database id of the page to read.
+     * @returns The HTML content as a UTF-8 string, or `null` when no body
+     *   is stored for `pageId`.
+     * @example
+     * const html = await accessor.getHtmlOfPage(pageId);
+     * if (html === null) {
+     *   // page has no stored body — redirect source / non-HTML / failed render
+     * } else {
+     *   processHtml(html);
+     * }
      */
-    getHtmlOfPage(filePath: string | null, openZipped?: boolean): Promise<string | null>;
+    getHtmlOfPage(pageId: number): Promise<string | null>;
+    /**
+     * Retrieves the JSON-LD / SpeculationRules entries for the given page,
+     * parsed back from the `page_jsonld` table.
+     * @param pageId - The database id of the page.
+     * @returns Ordered entries with `kind`, `type`, `raw`, `parsed`, `parseError`.
+     */
+    getJsonLdOfPage(pageId: number): Promise<import("./meta/types.js").JsonLdRow[]>;
     /**
      * Returns the underlying Knex query builder instance for direct SQL access.
      * Enables advanced queries (GROUP BY, HAVING, JOINs) at the database layer
@@ -106,6 +172,13 @@ export declare class ArchiveAccessor extends EventEmitter<DatabaseEvent> {
      * @returns An array of resource URL strings.
      */
     getResourceUrlList(): Promise<any[]>;
+    /**
+     * Retrieves the Wappalyzer tag entries for the given page, parsed back
+     * from the `page_tags` table.
+     * @param pageId - The database id of the page.
+     * @returns Ordered entries with provider, category, externalId, etc.
+     */
+    getTagsOfPage(pageId: number): Promise<import("./meta/types.js").TagRow[]>;
     /**
      * Stores custom data in the archive under the configured namespace.
      * Requires a namespace to be set on this accessor; throws if namespace is null.

package/lib/archive/archive-accessor.js

@@ -1,8 +1,6 @@
 import path from 'node:path';
-import { extractZip, unzip } from '@d-zero/fs/zip';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
 import { log } from './debug.js';
-import { exists } from './filesystem/exists.js';
 import { outputJSON } from './filesystem/output-json.js';
 import { outputText } from './filesystem/output-text.js';
 import { readJSON } from './filesystem/read-json.js';
@@ -17,13 +15,45 @@ import { safePath } from './safe-path.js';
  * by `Archive.connect` for read-only access to an existing archive.
  * It supports querying pages, anchors, referrers, resources, and custom data.
  */
+/**
+ * Default timeout for {@link ArchiveAccessor.close}'s `db.destroy()` step.
+ *
+ * `knex.destroy()` will otherwise wait the full `acquireTimeoutMillis`
+ * (10 minutes in this repo) for in-flight queries to drain. For a viewer
+ * shut down by Ctrl-C while the live crawler holds a long write lock that
+ * is an unacceptable user experience, so we bound the wait and treat the
+ * accessor as closed after the timeout regardless.
+ */
+const DEFAULT_CLOSE_TIMEOUT_MS = 5000;
 export class ArchiveAccessor extends EventEmitter {
+    /**
+     * Promise tracking an in-progress (or completed) close. `null` means the
+     * accessor is open and idle; a settled promise means we are closed (the
+     * accessor stays "closed" even if `db.destroy()` rejected, because there
+     * is nothing safe to retry — see {@link close}).
+     */
+    #closeOnce = null;
     /** The SQLite database instance for querying archived data. */
     #db;
     /** Namespace prefix for custom data storage (e.g. `"analysis/plugin-name"`). `null` disables `setData`. */
     #namespace = null;
+    /**
+     * Whether this accessor was opened in read-only mode. With HTML stored
+     * as a SQLite BLOB, this no longer toggles any code path — the SELECT
+     * is identical for writer- and reader-mode accessors. Kept on the
+     * accessor so callers like the viewer can still surface "this archive
+     * is being read read-only" UI hints without re-deriving it.
+     */
+    #readOnly;
     /** Absolute path to the temporary working directory containing the database and files. */
     #tmpDir;
+    /**
+     * Whether this accessor was opened in read-only mode (no filesystem
+     * mutation on `tmpDir`).
+     */
+    get readOnly() {
+        return this.#readOnly;
+    }
     /**
      * The absolute path to the temporary working directory used by this accessor.
      */
@@ -36,16 +66,59 @@ export class ArchiveAccessor extends EventEmitter {
      * @param db - The Database instance for querying the SQLite database.
      * @param namespace - An optional namespace for scoping custom data storage.
      *                    When null, `setData` is not available.
+     * @param options - Construction options.
+     * @param options.readOnly - When `true`, helpers must not mutate the
+     *   filesystem under `tmpDir` (used for live-crawl / stub-mode opens
+     *   where any write would race the crawler).
      */
-    constructor(tmpDir, db, namespace = null) {
+    constructor(tmpDir, db, namespace = null, options = {}) {
         super();
         this.#tmpDir = tmpDir;
         this.#db = db;
         this.#namespace = namespace;
+        this.#readOnly = options.readOnly ?? false;
         this.#db.on('error', (e) => {
             void this.emit('error', e);
         });
     }
+    /**
+     * Closes the underlying database connection.
+     *
+     * This is the **read-only** close path: it releases the SQLite handle and
+     * does nothing else. The temporary working directory is left untouched and
+     * no `.nitpicker` archive is produced. This makes it safe to call from
+     * read-only consumers (e.g. the viewer attached to an in-progress crawl's
+     * tmpDir), where touching the filesystem would race with — or destroy —
+     * the live crawler's working state.
+     *
+     * Subclasses that own the archive's lifecycle (notably {@link Archive})
+     * override this to add write/cleanup steps.
+     *
+     * **Idempotent and concurrent-safe**: the first invocation captures the
+     * close promise; later invocations (from the same caller, a shutdown
+     * signal handler, or a parallel manager teardown) await the same
+     * promise and resolve together. If `db.destroy()` rejects, the
+     * rejection propagates to *all* awaiters and the accessor stays
+     * latched closed — a hung knex pool is not safe to "retry close".
+     *
+     * **Bounded**: when the optional `timeoutMs` (default {@link
+     * DEFAULT_CLOSE_TIMEOUT_MS}) elapses before `db.destroy()` settles, the
+     * call resolves with a warning. This prevents a viewer shutdown from
+     * being held for the underlying pool's 10-minute `acquireTimeoutMillis`
+     * when the live crawler holds the SQLite write lock.
+     * @param options - Close options.
+     * @param options.timeoutMs - Milliseconds to wait for `db.destroy()`
+     *   before giving up. Use `Infinity` to wait indefinitely (only
+     *   advisable in tests and batch jobs that own the DB exclusively).
+     */
+    async close(options = {}) {
+        if (this.#closeOnce) {
+            return this.#closeOnce;
+        }
+        const timeoutMs = options.timeoutMs ?? DEFAULT_CLOSE_TIMEOUT_MS;
+        this.#closeOnce = this.#runClose(timeoutMs);
+        return this.#closeOnce;
+    }
     /**
      * Retrieves anchor (link) data for a specific page by its database ID.
      * @param pageId - The database ID of the page whose anchors to retrieve.
@@ -71,43 +144,50 @@ export class ArchiveAccessor extends EventEmitter {
         return await readText(filePath);
     }
     /**
-     * Reads the HTML content of a page snapshot from the archive.
-     * Supports reading from both unzipped directories and zipped snapshot archives.
-     * @param filePath - The relative file path to the HTML snapshot, or null.
-     * @param openZipped - Whether to attempt unzipping the snapshot archive. Defaults to `true`.
-     * @returns The HTML content as a string, or null if the snapshot is not found or filePath is null.
+     * Reads the HTML snapshot of a page from the archive.
+     *
+     * HTML is stored as zstd-compressed BLOBs in `page_html_blobs` (keyed by
+     * SHA-256 of the raw bytes) with `page_html_ref` linking `page_id → hash`.
+     * The read is a straight join + decompress; writer- and read-only (stub)
+     * accessors take the same code path because nothing here touches the
+     * filesystem.
+     *
+     * Returns `null` when the page row exists but has no stored body — for
+     * example a redirect source, a non-HTML resource (PDF), a page that
+     * failed to render, or an external page (whose row is metadata-only).
+     * Distinguishing "no body stored" from "empty body" is preserved: an
+     * empty HTML string returns `''`, not `null`.
+     *
+     * Throws if the page's referenced blob is missing or the codec marker
+     * is unrecognised — both indicate an archive that was truncated or
+     * written by a future tool, neither of which we silently paper over.
+     * @param pageId - The database id of the page to read.
+     * @returns The HTML content as a UTF-8 string, or `null` when no body
+     *   is stored for `pageId`.
+     * @example
+     * const html = await accessor.getHtmlOfPage(pageId);
+     * if (html === null) {
+     *   // page has no stored body — redirect source / non-HTML / failed render
+     * } else {
+     *   processHtml(html);
+     * }
      */
-    async getHtmlOfPage(filePath, openZipped = true) {
-        if (!filePath) {
-            return null;
-        }
-        const snapshotDir = safePath(this.#tmpDir, path.dirname(filePath));
-        const name = path.basename(filePath);
-        if (openZipped) {
-            await unzip(`${snapshotDir}.zip`, snapshotDir);
-        }
-        if (exists(snapshotDir)) {
-            log('Load %s directly because snapshot dir is unzipped', name);
-            const html = await readText(path.resolve(snapshotDir, name)).catch((error) => error);
-            if (typeof html === 'string') {
-                log('Loaded: %s ...', html.split('\n')[0]);
-                return html;
-            }
-            log('Failed Loading: %O', html);
-            return null;
+    async getHtmlOfPage(pageId) {
+        const html = await this.#db.getHtmlOfPageById(pageId);
+        if (html === null) {
+            log('No HTML body stored for page id=%d', pageId);
         }
-        log('Extracts %s from zipped snapshots', name);
-        const zipDir = await extractZip(`${snapshotDir}.zip`);
-        const file = zipDir.files.find((f) => f.type === 'File' && f.path === name);
-        if (!file) {
-            log('Failed: Not found %s from zipped snapshots', name);
-            return null;
-        }
-        const buffer = await file.buffer();
-        const html = buffer.toString('utf8') || null;
-        log('Succeeded: Extracts %s from zipped snapshots', name);
         return html;
     }
+    /**
+     * Retrieves the JSON-LD / SpeculationRules entries for the given page,
+     * parsed back from the `page_jsonld` table.
+     * @param pageId - The database id of the page.
+     * @returns Ordered entries with `kind`, `type`, `raw`, `parsed`, `parseError`.
+     */
+    async getJsonLdOfPage(pageId) {
+        return this.#db.getJsonLdOfPage(pageId);
+    }
     /**
      * Returns the underlying Knex query builder instance for direct SQL access.
      * Enables advanced queries (GROUP BY, HAVING, JOINs) at the database layer
@@ -200,6 +280,15 @@ export class ArchiveAccessor extends EventEmitter {
     async getResourceUrlList() {
         return this.#db.getResourceUrlList();
     }
+    /**
+     * Retrieves the Wappalyzer tag entries for the given page, parsed back
+     * from the `page_tags` table.
+     * @param pageId - The database id of the page.
+     * @returns Ordered entries with provider, category, externalId, etc.
+     */
+    async getTagsOfPage(pageId) {
+        return this.#db.getTagsOfPage(pageId);
+    }
     /**
      * Stores custom data in the archive under the configured namespace.
      * Requires a namespace to be set on this accessor; throws if namespace is null.
@@ -282,4 +371,41 @@ export class ArchiveAccessor extends EventEmitter {
         log('Create Page Data: Done');
         return pPages;
     }
+    /**
+     * Actual close worker — invoked exactly once per accessor via
+     * {@link close}'s shared promise. Races `db.destroy()` against the
+     * caller-supplied timeout; on timeout we log and resolve so the
+     * consumer (typically a process shutting down) is not blocked, even
+     * though the underlying knex pool may still be draining in the
+     * background.
+     * @param timeoutMs - Maximum time to wait for `db.destroy()`.
+     */
+    async #runClose(timeoutMs) {
+        if (!Number.isFinite(timeoutMs)) {
+            await this.#db.destroy();
+            return;
+        }
+        let timer = null;
+        const timeout = new Promise((resolve) => {
+            timer = setTimeout(() => resolve('timeout'), timeoutMs);
+        });
+        // Track destroy() so we can attach an error-suppressing handler if we
+        // give up waiting — otherwise a late rejection becomes an unhandled
+        // promise rejection on the process.
+        const destroy = this.#db.destroy().then(() => 'done');
+        try {
+            const result = await Promise.race([destroy, timeout]);
+            if (result === 'timeout') {
+                log('ArchiveAccessor.close: db.destroy() did not settle within %dms — giving up', timeoutMs);
+                destroy.catch((error) => {
+                    log('ArchiveAccessor.close: late db.destroy() rejection (post-timeout): %O', error);
+                });
+            }
+        }
+        finally {
+            if (timer) {
+                clearTimeout(timer);
+            }
+        }
+    }
 }

package/lib/archive/archive.d.ts

@@ -1,12 +1,15 @@
-import type { Config } from './types.js';
+import type { Config, PageSource } from './types.js';
 import type { PageData, CrawlerError, Resource } from '../utils/types/types.js';
 import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
 import { ArchiveAccessor } from './archive-accessor.js';
 /**
- * Main archive class for creating, opening, resuming, and writing Nitpicker archive files (`.nitpicker`).
+ * Main archive class for creating, opening, resuming, and writing Nitpicker
+ * archive files (`.nitpicker`).
  *
- * An Archive wraps a SQLite database and optional HTML snapshots into a compressed
- * tar archive. It extends {@link ArchiveAccessor} to provide read access to stored data.
+ * An Archive wraps a single SQLite database into a tar archive. HTML
+ * bodies live inside the same DB as zstd-compressed BLOBs (see #75) — the
+ * tar payload is effectively just `db.sqlite`. It extends
+ * {@link ArchiveAccessor} to provide read access to stored data.
  *
  * Use the static factory methods ({@link Archive.create}, {@link Archive.open},
  * {@link Archive.resume}, {@link Archive.connect}) to obtain instances.
@@ -18,22 +21,42 @@ export default class Archive extends ArchiveAccessor {
      * The absolute file path of the archive (`.nitpicker` file).
      */
     get filePath(): string;
+    /**
+     * The intermediate directory `Archive.write()` produces by renaming
+     * `tmpDir` before tarring (`{cwd}/{archiveName}`). Exposed so the
+     * manager can include it in its cleanup-on-failure path: if `tar()`
+     * fails after the rename, this directory is orphaned and would
+     * otherwise be invisible to a `rmSync(tmpDir)` recovery.
+     */
+    get renamedDir(): string;
     private constructor();
     /**
      * @deprecated This method is no longer functional.
      */
     abort(): void;
     /**
-     * Appends an error entry to the archive's error log file.
+     * Records a crawler-level error to both the human-readable `error.log` (full
+     * stack, for debugging) and the structured `crawl_errors` table (queryable,
+     * for the `error-kinds` analysis). The cause is not classified here — it is
+     * derived on read. `error.log` keeps the full stack while `crawl_errors`
+     * stores `error.message`; both normally carry the same cause token (e.g.
+     * `ENOTFOUND`), so classification agrees across the two — only an error whose
+     * cause lives solely in deeper stack frames could differ.
      * @param error - The crawler error object containing process and URL information.
      */
     addError(error: CrawlerError): Promise<void>;
     /**
-     * Closes the archive. If the archive file does not yet exist on disk,
-     * it writes the archive first. If the temporary directory still exists,
-     * it is removed.
+     * Records a partial scrape failure against the page identified by `url`.
+     *
+     * The corresponding `pages` row is created on demand (or matched if it
+     * already exists), so the call works even if the page's normal data has
+     * not been written yet.
+     * @param url - URL of the affected page.
+     * @param phase - Scrape phase name (typically `'retryExhausted'`).
+     * @param message - Human-readable failure message.
+     * @param isExternal - Whether the URL is external. Defaults to `false`.
      */
-    close(): Promise<void>;
+    addPageError(url: string, phase: string, message: string, isExternal?: boolean): Promise<void>;
     /**
      * Retrieves the current crawling state, including lists of scraped and pending URLs.
      * @returns An object with `scraped` and `pending` URL arrays.
@@ -42,11 +65,56 @@ export default class Archive extends ArchiveAccessor {
         scraped: string[];
         pending: string[];
     }>;
+    /**
+     * Return the subset of `urls` that already exist as `pages.url`. Used by
+     * `CrawlerOrchestrator.inventory` to filter the user-supplied URL list
+     * down to "URLs that are NOT yet in the archive" — only those reach the
+     * HEAD / scrape pipeline. Existing URLs are skipped to keep the second
+     * (and N-th) `--inventory` pass non-destructive.
+     * @param urls - Candidate URLs in `withoutHashAndAuth` form.
+     * @returns URLs already present in `pages`.
+     */
+    getExistingPageUrls(urls: readonly string[]): Promise<string[]>;
+    /**
+     * Return the subset of `urls` that already exist as `resources.url`. See
+     * {@link Archive.getExistingPageUrls} — the resource-side counterpart used
+     * by inventory mode to skip URLs that are already tracked as
+     * sub-resources.
+     * @param urls - Candidate URLs.
+     * @returns URLs already present in `resources`.
+     */
+    getExistingResourceUrls(urls: readonly string[]): Promise<string[]>;
+    /**
+     * Retrieves a single recorded sub-resource by its URL.
+     * @param urls - URL candidates to match against the stored resource URL.
+     * @returns The raw resource row, or `null` if none match.
+     */
+    getResourceByUrl(urls: readonly string[]): Promise<import("./types.js").DB_Resource | null>;
+    /**
+     * Counts the number of pages already scraped as crawl targets in the archive.
+     *
+     * Lets the crawler initialize its session-progress counter on resume so the
+     * displayed HTML-page count accounts for previously-rendered pages.
+     * @returns The count of pages with `isTarget = 1` and `scraped = 1`.
+     */
+    getScrapedHtmlPageCount(): Promise<number>;
     /**
      * Retrieves the base URL of the crawl session from the archive database.
      * @returns The base URL string.
      */
     getUrl(): Promise<any>;
+    /**
+     * Releases the SQLite handle and the advisory lock **without** writing
+     * the archive or removing `tmpDir`.
+     *
+     * Use this when you need to detach from a freshly-created `Archive`
+     * without finalising it — fixtures producing a stub state for tests,
+     * tooling that wants to leave the tmpDir alive for `crawl --resume`,
+     * or any non-orchestrator caller that owns the lifecycle externally.
+     * Shares the same idempotency guard as {@link close}, so the two paths
+     * are mutually exclusive (the first one called wins).
+     */
+    releaseHandle(): Promise<void>;
     /**
      * Promote previously-external pages that now fall under the (possibly extended)
      * scope back to a pending state so that the crawler re-scrapes them as fully
@@ -56,30 +124,53 @@ export default class Archive extends ArchiveAccessor {
      * @returns The URLs that were repromoted.
      */
     repromoteExternalPages(scopes: ReadonlyMap<string, readonly ExURL[]>, options?: ParseURLOptions): Promise<string[]>;
+    /**
+     * Reset previously-failed pages back to pending so a follow-up crawl re-fetches them.
+     *
+     * Delegates to {@link Database.resetFailedPages}. See that method for the
+     * exact failure criteria (missing status / content type, or a 5xx status).
+     * @returns The URLs of the pages that were reset to pending.
+     */
+    resetFailedPages(): Promise<string[]>;
     /**
      * Stores the crawl configuration into the archive database.
      * @param config - The configuration object to store.
      */
     setConfig(config: Config): Promise<number[]>;
     /**
-     * Stores an external page's data in the archive database without saving a snapshot.
+     * Stores an external page's data in the archive database without storing
+     * an HTML snapshot. External-page rows carry only metadata (status, title,
+     * content-type), never a rendered body.
      * @param pageInfo - The page data to store.
+     * @param source - Provenance label for new rows. `undefined` leaves the DB DEFAULT (`'crawled'`).
      */
-    setExternalPage(pageInfo: PageData): Promise<void>;
+    setExternalPage(pageInfo: PageData, source?: PageSource): Promise<void>;
     /**
-     * Stores a crawled page's data in the archive database and optionally saves an HTML snapshot.
-     * If the snapshot file write fails, the HTML path in the database is cleared to prevent
-     * referencing a non-existent file, and the error is re-thrown.
+     * Stores a crawled page's data in the archive database, persisting the
+     * rendered HTML body as a zstd-compressed BLOB inside the same SQLite
+     * transaction. Storage is content-addressable: identical bodies across
+     * pages share a single `page_html_blobs` row.
      * @param pageInfo - The page data to store.
+     * @param source - Provenance label for new rows. `undefined` leaves the DB DEFAULT (`'crawled'`).
      * @returns The database ID of the stored page.
-     * @throws {Error} Re-throws any error from the snapshot file write after clearing the HTML path.
      */
-    setPage(pageInfo: PageData): Promise<number>;
+    setPage(pageInfo: PageData, source?: PageSource): Promise<number>;
+    /**
+     * Records a redirect edge without re-storing the destination's content.
+     *
+     * The crawler calls this (instead of {@link setPage}) when a URL redirects to
+     * a destination that has already been rendered (#73): only the source →
+     * destination edge is written, leaving the destination's stored title / meta /
+     * anchors / images untouched.
+     * @param pageInfo - The HEAD-resolved page data carrying the redirect chain.
+     */
+    setRedirect(pageInfo: PageData): Promise<void>;
     /**
      * Stores a sub-resource (CSS, JS, image, etc.) in the archive database.
      * @param resource - The resource data to store.
+     * @param source - Provenance label for new rows. `undefined` leaves the DB DEFAULT (`'crawled'`).
      */
-    setResources(resource: Resource): Promise<void>;
+    setResources(resource: Resource, source?: PageSource): Promise<void>;
     /**
      * Stores the referrer relationship between a resource and the page that references it.
      * @param params - An object containing `url` (the page URL) and `src` (the resource URL).
@@ -109,25 +200,35 @@ export default class Archive extends ArchiveAccessor {
      */
     updateConfig(patch: Partial<Config>): Promise<void>;
     /**
-     * Writes the archive to disk as a compressed `.nitpicker` file.
+     * Writes the archive to disk as a `.nitpicker` tar file.
      *
-     * This method compresses the HTML snapshot directory into a zip file,
-     * renames the temporary working directory, and creates the final tar archive.
-     * The temporary directory is removed after writing.
+     * Checkpoints the SQLite WAL so the database is self-contained inside
+     * `db.sqlite`, renames the temporary working directory to the archive's
+     * basename, and tars it into the final `.nitpicker`. The tar container
+     * holds a single `db.sqlite` file (the legacy `snapshot-html.zip` is gone
+     * — HTML lives as BLOBs in the DB), so finalisation is effectively a
+     * single-file copy with no per-snapshot syscalls.
      */
     write(): Promise<void>;
     /** The file extension for Nitpicker archive files (without the leading dot). */
     static FILE_EXTENSION: string;
-    /** The directory name used for storing HTML snapshots within the archive. */
-    static readonly SNAPSHOT_HTML_DIR = "snapshot-html";
     /** The filename of the SQLite database within the archive. */
     static readonly SQLITE_DB_FILE_NAME = "db.sqlite";
     /** The prefix used for temporary working directories during archive operations. */
     static TMP_DIR_PREFIX: string;
     /**
      * Opens a read-only connection to an existing archive's database.
+     *
      * Returns an {@link ArchiveAccessor} that provides query methods
-     * without the ability to modify or write the archive.
+     * without the ability to modify or write the archive. The DB is opened
+     * in **read-only mode**: no schema migrations run, and the connection
+     * refuses to resurrect a missing parent directory or db file (so a
+     * TOCTOU window between source classification and this call cannot
+     * silently produce an empty phantom tmpDir).
+     *
+     * The returned accessor is also marked read-only so consumer-facing
+     * helpers (e.g. {@link ArchiveAccessor.getHtmlOfPage}) avoid any
+     * filesystem mutation on the user's tmpDir.
      * @param tmpDir - The path to the temporary directory containing the database.
      * @param namespace - An optional namespace for scoping data access within the archive.
      * @returns An ArchiveAccessor instance for querying the archive data.
@@ -167,6 +268,28 @@ export default class Archive extends ArchiveAccessor {
      * @returns A formatted timestamp string.
      */
     static timestamp(): string;
+    /**
+     * Closes the archive. If the archive file does not yet exist on disk,
+     * it writes the archive first. If the temporary directory still exists,
+     * it is removed. The database connection is then closed via
+     * {@link ArchiveAccessor.close} (the base class owns the SQLite handle),
+     * and finally the archive's advisory lock is released.
+     *
+     * **Idempotent**: the first invocation captures the close promise;
+     * subsequent invocations (signal handlers, parallel teardowns, retried
+     * orchestrator paths) await the same promise instead of re-entering
+     * the destructive prologue on a half-mutated state. If the first
+     * close fails (e.g. ENOSPC during tar), the rejection propagates to
+     * all awaiters and the archive stays latched closed — there is no
+     * safe way to retry `write()` once `tmpDir` has been renamed.
+     *
+     * **Read-only consumers must not reach this override.** Anything that
+     * obtains an archive view via {@link Archive.connect} receives an
+     * {@link ArchiveAccessor} (not an `Archive`), so `close()` resolves to
+     * the safe base implementation — no `write()`, no `remove()`, no lock
+     * release — leaving the tmpDir intact for the live crawler.
+     */
+    close(): Promise<void>;
     /**
      * Retrieves the crawl configuration stored in the archive database.
      * @returns The configuration object.