@nitpicker/crawler 0.13.0 → 0.15.0

package/lib/archive/archive-accessor.d.ts

@@ -68,6 +68,27 @@ export declare class ArchiveAccessor extends EventEmitter<DatabaseEvent> {
      * @returns An array of anchor records found on the page.
      */
     getAnchorsOnPage(pageId: number): Promise<any[]>;
+    /**
+     * Retrieves the audios within the given page's detected main content
+     * region, from `page_main_content_audios`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered audio rows.
+     */
+    getAudiosOfPage(pageId: number): Promise<import("./meta/types.js").MainContentAudioRow[]>;
+    /**
+     * Retrieves the button-like elements within the given page's detected
+     * main content region, from `page_main_content_buttons`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered button rows.
+     */
+    getButtonsOfPage(pageId: number): Promise<import("./meta/types.js").MainContentButtonRow[]>;
+    /**
+     * Retrieves the canvases within the given page's detected main content
+     * region, from `page_main_content_canvases`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered canvas rows.
+     */
+    getCanvasesOfPage(pageId: number): Promise<import("./meta/types.js").MainContentCanvasRow[]>;
     /**
      * Retrieves the crawl configuration stored in the archive database.
      * @returns The parsed {@link Config} object.
@@ -87,6 +108,13 @@ export declare class ArchiveAccessor extends EventEmitter<DatabaseEvent> {
      * @returns The file contents as a string.
      */
     getData(name: string, format?: 'txt' | 'html'): Promise<string>;
+    /**
+     * Retrieves the headings within the given page's detected main content
+     * region, from `page_main_content_headings`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered heading rows.
+     */
+    getHeadingsOfPage(pageId: number): Promise<import("./meta/types.js").MainContentHeadingRow[]>;
     /**
      * Reads the HTML snapshot of a page from the archive.
      *
@@ -117,6 +145,13 @@ export declare class ArchiveAccessor extends EventEmitter<DatabaseEvent> {
      * }
      */
     getHtmlOfPage(pageId: number): Promise<string | null>;
+    /**
+     * Retrieves the iframes within the given page's detected main content
+     * region, from `page_main_content_iframes`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered iframe rows.
+     */
+    getIframesOfPage(pageId: number): Promise<import("./meta/types.js").MainContentIframeRow[]>;
     /**
      * Retrieves the JSON-LD / SpeculationRules entries for the given page,
      * parsed back from the `page_jsonld` table.
@@ -131,6 +166,20 @@ export declare class ArchiveAccessor extends EventEmitter<DatabaseEvent> {
      * @returns The Knex instance connected to the SQLite database.
      */
     getKnex(): import("knex").Knex<any, any[]>;
+    /**
+     * Retrieves the images within the given page's detected main content
+     * region, from `page_main_content_images`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered image rows.
+     */
+    getMainContentImagesOfPage(pageId: number): Promise<import("./meta/types.js").MainContentImageRow[]>;
+    /**
+     * Retrieves the tables within the given page's detected main content
+     * region, from `page_main_content_tables`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered table rows.
+     */
+    getMainContentTablesOfPage(pageId: number): Promise<import("./meta/types.js").MainContentTableRow[]>;
     /**
      * Retrieves all pages from the archive, optionally filtered by type.
      * Eagerly loads redirect relationships (`redirectFrom`) but does NOT load
@@ -179,6 +228,13 @@ export declare class ArchiveAccessor extends EventEmitter<DatabaseEvent> {
      * @returns Ordered entries with provider, category, externalId, etc.
      */
     getTagsOfPage(pageId: number): Promise<import("./meta/types.js").TagRow[]>;
+    /**
+     * Retrieves the videos within the given page's detected main content
+     * region, from `page_main_content_videos`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered video rows.
+     */
+    getVideosOfPage(pageId: number): Promise<import("./meta/types.js").MainContentVideoRow[]>;
     /**
      * Stores custom data in the archive under the configured namespace.
      * Requires a namespace to be set on this accessor; throws if namespace is null.

package/lib/archive/archive-accessor.js

@@ -128,6 +128,33 @@ export class ArchiveAccessor extends EventEmitter {
         const refs = await this.#db.getAnchorsOnPage(pageId);
         return refs;
     }
+    /**
+     * Retrieves the audios within the given page's detected main content
+     * region, from `page_main_content_audios`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered audio rows.
+     */
+    async getAudiosOfPage(pageId) {
+        return this.#db.getAudiosOfPage(pageId);
+    }
+    /**
+     * Retrieves the button-like elements within the given page's detected
+     * main content region, from `page_main_content_buttons`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered button rows.
+     */
+    async getButtonsOfPage(pageId) {
+        return this.#db.getButtonsOfPage(pageId);
+    }
+    /**
+     * Retrieves the canvases within the given page's detected main content
+     * region, from `page_main_content_canvases`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered canvas rows.
+     */
+    async getCanvasesOfPage(pageId) {
+        return this.#db.getCanvasesOfPage(pageId);
+    }
     /**
      * Retrieves the crawl configuration stored in the archive database.
      * @returns The parsed {@link Config} object.
@@ -143,6 +170,15 @@ export class ArchiveAccessor extends EventEmitter {
         }
         return await readText(filePath);
     }
+    /**
+     * Retrieves the headings within the given page's detected main content
+     * region, from `page_main_content_headings`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered heading rows.
+     */
+    async getHeadingsOfPage(pageId) {
+        return this.#db.getHeadingsOfPage(pageId);
+    }
     /**
      * Reads the HTML snapshot of a page from the archive.
      *
@@ -179,6 +215,15 @@ export class ArchiveAccessor extends EventEmitter {
         }
         return html;
     }
+    /**
+     * Retrieves the iframes within the given page's detected main content
+     * region, from `page_main_content_iframes`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered iframe rows.
+     */
+    async getIframesOfPage(pageId) {
+        return this.#db.getIframesOfPage(pageId);
+    }
     /**
      * Retrieves the JSON-LD / SpeculationRules entries for the given page,
      * parsed back from the `page_jsonld` table.
@@ -197,6 +242,24 @@ export class ArchiveAccessor extends EventEmitter {
     getKnex() {
         return this.#db.getKnex();
     }
+    /**
+     * Retrieves the images within the given page's detected main content
+     * region, from `page_main_content_images`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered image rows.
+     */
+    async getMainContentImagesOfPage(pageId) {
+        return this.#db.getMainContentImagesOfPage(pageId);
+    }
+    /**
+     * Retrieves the tables within the given page's detected main content
+     * region, from `page_main_content_tables`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered table rows.
+     */
+    async getMainContentTablesOfPage(pageId) {
+        return this.#db.getMainContentTablesOfPage(pageId);
+    }
     /**
      * Retrieves all pages from the archive, optionally filtered by type.
      * Eagerly loads redirect relationships (`redirectFrom`) but does NOT load
@@ -289,6 +352,15 @@ export class ArchiveAccessor extends EventEmitter {
     async getTagsOfPage(pageId) {
         return this.#db.getTagsOfPage(pageId);
     }
+    /**
+     * Retrieves the videos within the given page's detected main content
+     * region, from `page_main_content_videos`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered video rows.
+     */
+    async getVideosOfPage(pageId) {
+        return this.#db.getVideosOfPage(pageId);
+    }
     /**
      * Stores custom data in the archive under the configured namespace.
      * Requires a namespace to be set on this accessor; throws if namespace is null.

package/lib/archive/archive.d.ts

@@ -1,14 +1,20 @@
-import type { Config, InventoryRunMeta, PageSource } from './types.js';
+import type { TemplateClusterReason } from './db-ops/analysis/types.js';
+import type { Config, InsertNetworkOutageParams, InventoryRunMeta, PageSource } from './types.js';
+import type { OutageWindow } from '../is-within-outage-window.js';
 import type { PageData, CrawlerError, Resource } from '../utils/types/types.js';
+import type { ConsoleLogEntry } from '@d-zero/beholder';
 import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
 import { ArchiveAccessor } from './archive-accessor.js';
 /**
  * Main archive class for creating, opening, resuming, and writing Nitpicker
  * archive files (`.nitpicker`).
  *
- * An Archive wraps a single SQLite database into a tar archive. HTML
- * bodies live inside the same DB as zstd-compressed BLOBs (see #75) — the
- * tar payload is effectively just `db.sqlite`. It extends
+ * An Archive wraps a SQLite database into a tar archive. HTML bodies live
+ * inside the same DB as zstd-compressed BLOBs (see #75), so `db.sqlite` is
+ * normally the tar's only entry — but it is not the only entry the format
+ * allows: {@link ArchiveAccessor.setData} (namespace-scoped analyze output)
+ * and {@link Archive.saveInventorySourceList} (a saved `--inventory`
+ * source list) add plain files alongside it. It extends
  * {@link ArchiveAccessor} to provide read access to stored data.
  *
  * Use the static factory methods ({@link Archive.create}, {@link Archive.open},
@@ -66,6 +72,14 @@ export default class Archive extends ArchiveAccessor {
      * @param isExternal - Whether the URL is external. Defaults to `false`.
      */
     addPageError(url: string, phase: string, message: string, isExternal?: boolean): Promise<void>;
+    /**
+     * Closes an outage row by stamping `ended_at` — a no-op if already closed.
+     *
+     * Thin facade over {@link Database.closeNetworkOutage}.
+     * @param id - The `network_outages.id` to close.
+     * @param endedAt - Epoch ms the outage is considered to have ended.
+     */
+    closeNetworkOutage(id: number, endedAt: number): Promise<void>;
     /**
      * Retrieves the current crawling state, including lists of scraped and pending URLs.
      * @returns An object with `scraped` and `pending` URL arrays.
@@ -151,6 +165,15 @@ export default class Archive extends ArchiveAccessor {
      * @param urls - HTML seed URLs to pre-insert. No-op when empty.
      */
     insertInventorySeeds(urls: readonly ExURL[]): Promise<void>;
+    /**
+     * Appends one open row to the `network_outages` journal.
+     *
+     * Thin facade over {@link Database.insertNetworkOutage} — see
+     * {@link recordInventoryRun}'s docstring for why this indirection exists.
+     * @param params - The confirmed-outage fields to record.
+     * @returns The autoincremented `id` of the inserted row.
+     */
+    insertNetworkOutage(params: InsertNetworkOutageParams): Promise<number>;
     /**
      * Hostnames whose `crawl_errors` history is consistently DNS failures and
      * for which no recent 2xx/3xx page or resource is recorded. Consumed by
@@ -165,6 +188,13 @@ export default class Archive extends ArchiveAccessor {
      * @returns Lower-cased hostnames safe to short-circuit.
      */
     listDnsBurnedHostCandidates(): Promise<string[]>;
+    /**
+     * Lists every recorded outage as a resolved {@link OutageWindow}.
+     *
+     * Thin facade over {@link Database.listNetworkOutages}.
+     * @returns Resolved outage windows, or `[]` if none have been recorded.
+     */
+    listNetworkOutages(): Promise<OutageWindow[]>;
     /**
      * Appends one row to the `inventory_runs` audit log.
      *
@@ -206,6 +236,21 @@ export default class Archive extends ArchiveAccessor {
         line?: number | null;
         col?: number | null;
     }[]): Promise<void>;
+    /**
+     * Replaces the archive's DOM-structure template classification
+     * (`--templates`) with a fresh SQL-backed set.
+     *
+     * Thin facade over {@link Database.replacePageTemplates}; kept on
+     * `Archive` so the analyze pipeline can persist template keys without
+     * reaching into the low-level database class directly.
+     * @param templateKeysByUrl - Page URL → template key, as produced by
+     *   `@nitpicker/core`'s `classifyPageTemplates`.
+     * @param clusterReasonsByTemplateKey - Template key → cluster-selection
+     *   evidence, if the caller captured it. Omitting this always clears the
+     *   previously-stored reasons too — "no reason" means "not captured for
+     *   this run", never "carry over the previous run's reasons".
+     */
+    replacePageTemplates(templateKeysByUrl: ReadonlyMap<string, string>, clusterReasonsByTemplateKey?: ReadonlyMap<string, TemplateClusterReason>): Promise<void>;
     /**
      * Promote previously-external pages that now fall under the (possibly extended)
      * scope back to a pending state so that the crawler re-scrapes them as fully
@@ -223,11 +268,46 @@ export default class Archive extends ArchiveAccessor {
      * @returns The URLs of the pages that were reset to pending.
      */
     resetFailedPages(): Promise<string[]>;
+    /**
+     * Persists the raw bytes of an `--inventory` source URL list into the
+     * archive's tar payload, at `inventory/<sha256>.txt`.
+     *
+     * The file name is the content hash rather than the original file name:
+     * re-applying the same list is then a no-op write (`fs.writeFile`
+     * overwrites identical bytes), and the original name — which may embed a
+     * client/project identifier — is never retained (the archive already
+     * omits the source file's absolute path for the same reason; see
+     * `CrawlerOrchestrator.inventory`'s `source` param).
+     *
+     * This bypasses the namespace-scoped {@link ArchiveAccessor.setData} API
+     * (that one is reserved for analyze plugins and requires a namespace) —
+     * this always lands under the fixed `inventory/` prefix regardless of
+     * how this accessor was constructed. Callers that need to read the
+     * saved list back can use the inherited `getData(`inventory/${sha256}`,
+     * 'txt')`, since it resolves to the same path when no namespace is set.
+     *
+     * No entry is ever removed here — same accepted gap as `page_html_blobs`
+     * (a future #23 GC pass will sweep unreachable hashes across both). A
+     * source list that differs byte-for-byte on every run (e.g. a
+     * regenerated doc-root export with fresh timestamps) adds one entry per
+     * run with no pruning of superseded ones.
+     * @param sha256 - Lower-case hex SHA-256 digest of `bytes` (used as the file name).
+     * @param bytes - The exact bytes of the source list file, written verbatim.
+     */
+    saveInventorySourceList(sha256: string, bytes: Buffer): Promise<void>;
     /**
      * Stores the crawl configuration into the archive database.
      * @param config - The configuration object to store.
      */
     setConfig(config: Config): Promise<number[]>;
+    /**
+     * Replaces one page's captured console messages / page errors in the
+     * archive database.
+     * @param pageUrl - The originally-requested URL, normalised (`withoutHashAndAuth` form).
+     * @param redirectPaths - The redirect chain hops captured during fetch, in order.
+     * @param entries - The console log entries to persist.
+     */
+    setConsoleLogs(pageUrl: string, redirectPaths: readonly string[], entries: readonly ConsoleLogEntry[]): Promise<void>;
     /**
      * Stores an external page's data in the archive database without storing
      * an HTML snapshot. External-page rows carry only metadata (status, title,
@@ -300,10 +380,17 @@ export default class Archive extends ArchiveAccessor {
      *
      * Checkpoints the SQLite WAL so the database is self-contained inside
      * `db.sqlite`, renames the temporary working directory to the archive's
-     * basename, and tars it into the final `.nitpicker`. The tar container
-     * holds a single `db.sqlite` file (the legacy `snapshot-html.zip` is gone
-     * — HTML lives as BLOBs in the DB), so finalisation is effectively a
-     * single-file copy with no per-snapshot syscalls.
+     * basename, and tars the **entire tmpDir**. `db.sqlite` is normally the
+     * only entry (HTML lives as BLOBs in the DB, not a `snapshot-html.zip`),
+     * but a namespace-scoped `setData` write (analyze output) or
+     * `saveInventorySourceList` (a saved `--inventory` source list) adds
+     * extra files under tmpDir that get tarred right alongside it.
+     *
+     * This is why every writer path that reaches `write()` must open with
+     * `openPluginData: true` — `Archive.open`'s default extracts only
+     * `db.sqlite`, so a re-crawl (`append` / `inventory` / `retryFailed`)
+     * opened without it would tar back a tmpDir missing those extra files,
+     * silently dropping them from the rewritten archive.
      */
     write(): Promise<void>;
     /** The file extension for Nitpicker archive files (without the leading dot). */
@@ -471,7 +558,19 @@ type ArchiveOptions = {
  * Additional options for opening an existing archive.
  */
 type ArchiveOpenOptions = {
-    /** When true, extracts all files including plugin data. When false, only extracts the database and snapshots. */
+    /**
+     * When `false` (the default), only `db.sqlite` is extracted into tmpDir.
+     * When `true`, every tar entry is extracted, including non-namespace
+     * files written via {@link ArchiveAccessor.setData} (analyze output) or
+     * {@link Archive.saveInventorySourceList} (a saved `--inventory` source
+     * list).
+     *
+     * Every writer path that later calls {@link Archive.write} MUST pass
+     * `true`: `write()` re-tars whatever is currently in tmpDir, so a
+     * re-crawl (`append` / `inventory` / `retryFailed`) opened with the
+     * default would tar back a tmpDir missing those extra files, silently
+     * dropping them from the rewritten archive.
+     */
     openPluginData?: boolean;
 };
 export {};

package/lib/archive/archive.js

@@ -10,18 +10,23 @@ import { dbLog, log, saveLog } from './debug.js';
 import { appendText } from './filesystem/append-text.js';
 import { exists } from './filesystem/exists.js';
 import { isDir } from './filesystem/is-dir.js';
+import { outputBinary } from './filesystem/output-binary.js';
 import { peekTarTopDir } from './filesystem/peek-tar-top-dir.js';
 import { remove } from './filesystem/remove.js';
 import { rename } from './filesystem/rename.js';
 import { tar } from './filesystem/tar.js';
 import { untar } from './filesystem/untar.js';
+import { safePath } from './safe-path.js';
 /**
  * Main archive class for creating, opening, resuming, and writing Nitpicker
  * archive files (`.nitpicker`).
  *
- * An Archive wraps a single SQLite database into a tar archive. HTML
- * bodies live inside the same DB as zstd-compressed BLOBs (see #75) — the
- * tar payload is effectively just `db.sqlite`. It extends
+ * An Archive wraps a SQLite database into a tar archive. HTML bodies live
+ * inside the same DB as zstd-compressed BLOBs (see #75), so `db.sqlite` is
+ * normally the tar's only entry — but it is not the only entry the format
+ * allows: {@link ArchiveAccessor.setData} (namespace-scoped analyze output)
+ * and {@link Archive.saveInventorySourceList} (a saved `--inventory`
+ * source list) add plain files alongside it. It extends
  * {@link ArchiveAccessor} to provide read access to stored data.
  *
  * Use the static factory methods ({@link Archive.create}, {@link Archive.open},
@@ -119,6 +124,17 @@ export default class Archive extends ArchiveAccessor {
         dbLog('Add page error: %s [%s]', url, phase);
         await this.#db.insertPageError(url, phase, message, isExternal);
     }
+    /**
+     * Closes an outage row by stamping `ended_at` — a no-op if already closed.
+     *
+     * Thin facade over {@link Database.closeNetworkOutage}.
+     * @param id - The `network_outages.id` to close.
+     * @param endedAt - Epoch ms the outage is considered to have ended.
+     */
+    async closeNetworkOutage(id, endedAt) {
+        dbLog('Close network outage id=%d endedAt=%d', id, endedAt);
+        return await this.#db.closeNetworkOutage(id, endedAt);
+    }
     /**
      * Retrieves the current crawling state, including lists of scraped and pending URLs.
      * @returns An object with `scraped` and `pending` URL arrays.
@@ -227,6 +243,18 @@ export default class Archive extends ArchiveAccessor {
         dbLog('Insert inventory seeds: %d URL(s)', urls.length);
         await this.#db.insertInventorySeeds(urls.map((u) => u.withoutHashAndAuth));
     }
+    /**
+     * Appends one open row to the `network_outages` journal.
+     *
+     * Thin facade over {@link Database.insertNetworkOutage} — see
+     * {@link recordInventoryRun}'s docstring for why this indirection exists.
+     * @param params - The confirmed-outage fields to record.
+     * @returns The autoincremented `id` of the inserted row.
+     */
+    async insertNetworkOutage(params) {
+        dbLog('Insert network outage: startedAt=%d probeHost=%s', params.startedAt, params.probeHost);
+        return await this.#db.insertNetworkOutage(params);
+    }
     /**
      * Hostnames whose `crawl_errors` history is consistently DNS failures and
      * for which no recent 2xx/3xx page or resource is recorded. Consumed by
@@ -243,6 +271,15 @@ export default class Archive extends ArchiveAccessor {
     async listDnsBurnedHostCandidates() {
         return this.#db.listDnsBurnedHostCandidates();
     }
+    /**
+     * Lists every recorded outage as a resolved {@link OutageWindow}.
+     *
+     * Thin facade over {@link Database.listNetworkOutages}.
+     * @returns Resolved outage windows, or `[]` if none have been recorded.
+     */
+    async listNetworkOutages() {
+        return await this.#db.listNetworkOutages();
+    }
     /**
      * Appends one row to the `inventory_runs` audit log.
      *
@@ -286,6 +323,23 @@ export default class Archive extends ArchiveAccessor {
     async replaceAnalysisViolations(violations) {
         await this.#db.replaceAnalysisViolations(violations);
     }
+    /**
+     * Replaces the archive's DOM-structure template classification
+     * (`--templates`) with a fresh SQL-backed set.
+     *
+     * Thin facade over {@link Database.replacePageTemplates}; kept on
+     * `Archive` so the analyze pipeline can persist template keys without
+     * reaching into the low-level database class directly.
+     * @param templateKeysByUrl - Page URL → template key, as produced by
+     *   `@nitpicker/core`'s `classifyPageTemplates`.
+     * @param clusterReasonsByTemplateKey - Template key → cluster-selection
+     *   evidence, if the caller captured it. Omitting this always clears the
+     *   previously-stored reasons too — "no reason" means "not captured for
+     *   this run", never "carry over the previous run's reasons".
+     */
+    async replacePageTemplates(templateKeysByUrl, clusterReasonsByTemplateKey) {
+        await this.#db.replacePageTemplates(templateKeysByUrl, clusterReasonsByTemplateKey);
+    }
     /**
      * Promote previously-external pages that now fall under the (possibly extended)
      * scope back to a pending state so that the crawler re-scrapes them as fully
@@ -309,6 +363,36 @@ export default class Archive extends ArchiveAccessor {
         dbLog('Reset failed pages back to pending');
         return this.#db.resetFailedPages();
     }
+    /**
+     * Persists the raw bytes of an `--inventory` source URL list into the
+     * archive's tar payload, at `inventory/<sha256>.txt`.
+     *
+     * The file name is the content hash rather than the original file name:
+     * re-applying the same list is then a no-op write (`fs.writeFile`
+     * overwrites identical bytes), and the original name — which may embed a
+     * client/project identifier — is never retained (the archive already
+     * omits the source file's absolute path for the same reason; see
+     * `CrawlerOrchestrator.inventory`'s `source` param).
+     *
+     * This bypasses the namespace-scoped {@link ArchiveAccessor.setData} API
+     * (that one is reserved for analyze plugins and requires a namespace) —
+     * this always lands under the fixed `inventory/` prefix regardless of
+     * how this accessor was constructed. Callers that need to read the
+     * saved list back can use the inherited `getData(`inventory/${sha256}`,
+     * 'txt')`, since it resolves to the same path when no namespace is set.
+     *
+     * No entry is ever removed here — same accepted gap as `page_html_blobs`
+     * (a future #23 GC pass will sweep unreachable hashes across both). A
+     * source list that differs byte-for-byte on every run (e.g. a
+     * regenerated doc-root export with fresh timestamps) adds one entry per
+     * run with no pruning of superseded ones.
+     * @param sha256 - Lower-case hex SHA-256 digest of `bytes` (used as the file name).
+     * @param bytes - The exact bytes of the source list file, written verbatim.
+     */
+    async saveInventorySourceList(sha256, bytes) {
+        const filePath = safePath(this.tmpDir, 'inventory', `${sha256}.txt`);
+        await outputBinary(filePath, bytes);
+    }
     /**
      * Stores the crawl configuration into the archive database.
      * @param config - The configuration object to store.
@@ -317,6 +401,17 @@ export default class Archive extends ArchiveAccessor {
         dbLog('Set config: %O', config);
         return this.#db.setConfig(config);
     }
+    /**
+     * Replaces one page's captured console messages / page errors in the
+     * archive database.
+     * @param pageUrl - The originally-requested URL, normalised (`withoutHashAndAuth` form).
+     * @param redirectPaths - The redirect chain hops captured during fetch, in order.
+     * @param entries - The console log entries to persist.
+     */
+    async setConsoleLogs(pageUrl, redirectPaths, entries) {
+        dbLog('Set console logs: %d entries on %s', entries.length, pageUrl);
+        await this.#db.replaceConsoleLogs(pageUrl, redirectPaths, entries);
+    }
     /**
      * Stores an external page's data in the archive database without storing
      * an HTML snapshot. External-page rows carry only metadata (status, title,
@@ -410,10 +505,17 @@ export default class Archive extends ArchiveAccessor {
      *
      * Checkpoints the SQLite WAL so the database is self-contained inside
      * `db.sqlite`, renames the temporary working directory to the archive's
-     * basename, and tars it into the final `.nitpicker`. The tar container
-     * holds a single `db.sqlite` file (the legacy `snapshot-html.zip` is gone
-     * — HTML lives as BLOBs in the DB), so finalisation is effectively a
-     * single-file copy with no per-snapshot syscalls.
+     * basename, and tars the **entire tmpDir**. `db.sqlite` is normally the
+     * only entry (HTML lives as BLOBs in the DB, not a `snapshot-html.zip`),
+     * but a namespace-scoped `setData` write (analyze output) or
+     * `saveInventorySourceList` (a saved `--inventory` source list) adds
+     * extra files under tmpDir that get tarred right alongside it.
+     *
+     * This is why every writer path that reaches `write()` must open with
+     * `openPluginData: true` — `Archive.open`'s default extracts only
+     * `db.sqlite`, so a re-crawl (`append` / `inventory` / `retryFailed`)
+     * opened without it would tar back a tmpDir missing those extra files,
+     * silently dropping them from the rewritten archive.
      */
     async write() {
         saveLog('Starts: %s', this.#filePath);

package/lib/archive/body-hash/compute-body-hash.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Computes a content hash of a page's `<body>`, after normalizing away the
+ * kinds of incidental variance that would otherwise make two structurally
+ * identical pages hash differently: `/index.{ext}` URL-suffix forms and
+ * embedded dynamic tokens (cache-busting hashes, session/order ids, per-build
+ * CSS-module suffixes).
+ *
+ * Only the resulting hash is persisted (`page_meta.body_hash`) — the masked
+ * intermediate string is never stored. The unmasked original HTML remains
+ * fully recoverable from `page_html_blobs`, so nothing is lost by discarding
+ * it here.
+ * @param html - A full HTML document string (or fragment).
+ * @returns 32-byte SHA-256 hash of the masked `<body>` content, ready to
+ *   insert into a `BLOB` column.
+ * @example
+ * ```ts
+ * const hashA = computeBodyHash('<body><a href="/p/a1b2c3d4">x</a></body>');
+ * const hashB = computeBodyHash('<body><a href="/p/z9y8x7w6">x</a></body>');
+ * hashA.equals(hashB); // true — the differing token is masked before hashing
+ * ```
+ */
+export declare function computeBodyHash(html: string): Buffer;

package/lib/archive/body-hash/compute-body-hash.js

@@ -0,0 +1,31 @@
+import { computeContentHash } from '../populate-ref-tables/compute-content-hash.js';
+import { extractBody } from './extract-body.js';
+import { maskDynamicIds } from './mask-dynamic-ids.js';
+import { normalizeUrlLikeStrings } from './normalize-url-like-strings.js';
+/**
+ * Computes a content hash of a page's `<body>`, after normalizing away the
+ * kinds of incidental variance that would otherwise make two structurally
+ * identical pages hash differently: `/index.{ext}` URL-suffix forms and
+ * embedded dynamic tokens (cache-busting hashes, session/order ids, per-build
+ * CSS-module suffixes).
+ *
+ * Only the resulting hash is persisted (`page_meta.body_hash`) — the masked
+ * intermediate string is never stored. The unmasked original HTML remains
+ * fully recoverable from `page_html_blobs`, so nothing is lost by discarding
+ * it here.
+ * @param html - A full HTML document string (or fragment).
+ * @returns 32-byte SHA-256 hash of the masked `<body>` content, ready to
+ *   insert into a `BLOB` column.
+ * @example
+ * ```ts
+ * const hashA = computeBodyHash('<body><a href="/p/a1b2c3d4">x</a></body>');
+ * const hashB = computeBodyHash('<body><a href="/p/z9y8x7w6">x</a></body>');
+ * hashA.equals(hashB); // true — the differing token is masked before hashing
+ * ```
+ */
+export function computeBodyHash(html) {
+    const body = extractBody(html);
+    const normalized = normalizeUrlLikeStrings(body);
+    const masked = maskDynamicIds(normalized);
+    return computeContentHash(masked);
+}

package/lib/archive/body-hash/extract-body.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Extracts the inner HTML of the first `<body>` element from a full HTML
+ * document string.
+ *
+ * Uses a greedy match (`[\s\S]*`, not `[\s\S]*?`) so a literal `<body>`
+ * substring appearing inside the real body (e.g. an inline code sample) does
+ * not truncate the extracted content at that inner occurrence — the match
+ * always extends to the last `</body>` in the document.
+ *
+ * Falls back to returning the full input unchanged when no `<body>` tag is
+ * found (fragment HTML, a page that failed to render, or a snapshot cut off
+ * mid-render) rather than throwing, so callers never need a separate
+ * not-found branch.
+ * @param html - A full HTML document string, or a fragment.
+ * @returns The content between `<body...>` and `</body>`, or `html` unchanged
+ *   if no `<body>` tag is present.
+ */
+export declare function extractBody(html: string): string;

package/lib/archive/body-hash/extract-body.js

@@ -0,0 +1,29 @@
+// The opening tag's attribute span is matched attribute-aware
+// (`(?:"[^"]*"|'[^']*'|[^"'>])*`, not a plain `[^>]*`) so a literal `>`
+// inside a quoted attribute value (e.g. `<body data-x="a>b">`) — legal HTML,
+// since only the delimiting quote character itself must not appear
+// unescaped inside the value — does not end the match early. A plain
+// `[^>]*` would stop at that inner `>`, so the captured body would start
+// mid-attribute instead of at the real content.
+const BODY_PATTERN = /<body(?:"[^"]*"|'[^']*'|[^"'>])*>([\s\S]*)<\/body>/i;
+/**
+ * Extracts the inner HTML of the first `<body>` element from a full HTML
+ * document string.
+ *
+ * Uses a greedy match (`[\s\S]*`, not `[\s\S]*?`) so a literal `<body>`
+ * substring appearing inside the real body (e.g. an inline code sample) does
+ * not truncate the extracted content at that inner occurrence — the match
+ * always extends to the last `</body>` in the document.
+ *
+ * Falls back to returning the full input unchanged when no `<body>` tag is
+ * found (fragment HTML, a page that failed to render, or a snapshot cut off
+ * mid-render) rather than throwing, so callers never need a separate
+ * not-found branch.
+ * @param html - A full HTML document string, or a fragment.
+ * @returns The content between `<body...>` and `</body>`, or `html` unchanged
+ *   if no `<body>` tag is present.
+ */
+export function extractBody(html) {
+    const match = BODY_PATTERN.exec(html);
+    return match?.[1] ?? html;
+}