@nitpicker/crawler 0.14.0 → 0.16.0

package/lib/archive/archive.d.ts

@@ -1,14 +1,20 @@
-import type { Config, InventoryRunMeta, PageSource } from './types.js';
+import type { TemplateClusterReason } from './db-ops/analysis/types.js';
+import type { Config, InsertDedupeCapEventParams, InsertNetworkOutageParams, InventoryRunMeta, PageSource } from './types.js';
+import type { OutageWindow } from '../is-within-outage-window.js';
 import type { PageData, CrawlerError, Resource } from '../utils/types/types.js';
+import type { ConsoleLogEntry } from '@d-zero/beholder';
 import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
 import { ArchiveAccessor } from './archive-accessor.js';
 /**
  * Main archive class for creating, opening, resuming, and writing Nitpicker
  * archive files (`.nitpicker`).
  *
- * An Archive wraps a single SQLite database into a tar archive. HTML
- * bodies live inside the same DB as zstd-compressed BLOBs (see #75) — the
- * tar payload is effectively just `db.sqlite`. It extends
+ * An Archive wraps a SQLite database into a tar archive. HTML bodies live
+ * inside the same DB as zstd-compressed BLOBs (see #75), so `db.sqlite` is
+ * normally the tar's only entry — but it is not the only entry the format
+ * allows: {@link ArchiveAccessor.setData} (namespace-scoped analyze output)
+ * and {@link Archive.saveInventorySourceList} (a saved `--inventory`
+ * source list) add plain files alongside it. It extends
  * {@link ArchiveAccessor} to provide read access to stored data.
  *
  * Use the static factory methods ({@link Archive.create}, {@link Archive.open},
@@ -43,6 +49,17 @@ export default class Archive extends ArchiveAccessor {
      * @deprecated This method is no longer functional.
      */
     abort(): void;
+    /**
+     * Adds onto the `rejected_count` of a shape's `dedupe_cap_events` row,
+     * looked up by `shape_key` rather than `id` — used for a shape that
+     * capped in an earlier session (preloaded into `DedupeCapTracker`'s
+     * sticky set) and so has no event id from the current session.
+     *
+     * Thin facade over {@link Database.accumulateDedupeCapRejectedCount}.
+     * @param shapeKey - The capped shape whose rejection count to accumulate.
+     * @param rejectedCount - Additional anchors rejected for this shape in the current session.
+     */
+    accumulateDedupeCapRejectedCount(shapeKey: string, rejectedCount: number): Promise<void>;
     /**
      * Records a crawler-level error to both the human-readable `error.log` (full
      * stack, for debugging) and the structured `crawl_errors` table (queryable,
@@ -66,6 +83,23 @@ export default class Archive extends ArchiveAccessor {
      * @param isExternal - Whether the URL is external. Defaults to `false`.
      */
     addPageError(url: string, phase: string, message: string, isExternal?: boolean): Promise<void>;
+    /**
+     * Closes an outage row by stamping `ended_at` — a no-op if already closed.
+     *
+     * Thin facade over {@link Database.closeNetworkOutage}.
+     * @param id - The `network_outages.id` to close.
+     * @param endedAt - Epoch ms the outage is considered to have ended.
+     */
+    closeNetworkOutage(id: number, endedAt: number): Promise<void>;
+    /**
+     * Finalizes a `dedupe_cap_events` row by stamping `rejected_count` — a
+     * no-op if already finalized.
+     *
+     * Thin facade over {@link Database.finalizeDedupeCapEvent}.
+     * @param id - The `dedupe_cap_events.id` to finalize.
+     * @param rejectedCount - Number of anchors rejected for this shape after it capped.
+     */
+    finalizeDedupeCapEvent(id: number, rejectedCount: number): Promise<void>;
     /**
      * Retrieves the current crawling state, including lists of scraped and pending URLs.
      * @returns An object with `scraped` and `pending` URL arrays.
@@ -122,6 +156,15 @@ export default class Archive extends ArchiveAccessor {
      * @returns The base URL string.
      */
     getUrl(): Promise<string>;
+    /**
+     * Appends one row (`rejected_count = NULL`) to the `dedupe_cap_events`
+     * journal.
+     *
+     * Thin facade over {@link Database.insertDedupeCapEvent}.
+     * @param params - The newly-capped shape's fields to record.
+     * @returns The autoincremented `id` of the inserted row.
+     */
+    insertDedupeCapEvent(params: InsertDedupeCapEventParams): Promise<number>;
     /**
      * Pre-insert inventory non-HTML URLs as `source='inventory-seed'`
      * placeholders in the `resources` table — the non-HTML counterpart of
@@ -151,6 +194,24 @@ export default class Archive extends ArchiveAccessor {
      * @param urls - HTML seed URLs to pre-insert. No-op when empty.
      */
     insertInventorySeeds(urls: readonly ExURL[]): Promise<void>;
+    /**
+     * Appends one open row to the `network_outages` journal.
+     *
+     * Thin facade over {@link Database.insertNetworkOutage} — see
+     * {@link recordInventoryRun}'s docstring for why this indirection exists.
+     * @param params - The confirmed-outage fields to record.
+     * @returns The autoincremented `id` of the inserted row.
+     */
+    insertNetworkOutage(params: InsertNetworkOutageParams): Promise<number>;
+    /**
+     * Every distinct `dedupe_cap_events.shape_key` recorded in this archive.
+     * Consumed by `CrawlerOrchestrator` to preload `DedupeCapTracker`'s
+     * sticky set on `--resume` / `--append` / `--retry-failed` /
+     * `--inventory`, mirroring {@link listDnsBurnedHostCandidates}'s
+     * writer-only exposure.
+     * @returns Distinct shape keys already confirmed capped.
+     */
+    listDedupeCapShapeKeys(): Promise<string[]>;
     /**
      * Hostnames whose `crawl_errors` history is consistently DNS failures and
      * for which no recent 2xx/3xx page or resource is recorded. Consumed by
@@ -165,6 +226,13 @@ export default class Archive extends ArchiveAccessor {
      * @returns Lower-cased hostnames safe to short-circuit.
      */
     listDnsBurnedHostCandidates(): Promise<string[]>;
+    /**
+     * Lists every recorded outage as a resolved {@link OutageWindow}.
+     *
+     * Thin facade over {@link Database.listNetworkOutages}.
+     * @returns Resolved outage windows, or `[]` if none have been recorded.
+     */
+    listNetworkOutages(): Promise<OutageWindow[]>;
     /**
      * Appends one row to the `inventory_runs` audit log.
      *
@@ -215,8 +283,12 @@ export default class Archive extends ArchiveAccessor {
      * reaching into the low-level database class directly.
      * @param templateKeysByUrl - Page URL → template key, as produced by
      *   `@nitpicker/core`'s `classifyPageTemplates`.
+     * @param clusterReasonsByTemplateKey - Template key → cluster-selection
+     *   evidence, if the caller captured it. Omitting this always clears the
+     *   previously-stored reasons too — "no reason" means "not captured for
+     *   this run", never "carry over the previous run's reasons".
      */
-    replacePageTemplates(templateKeysByUrl: ReadonlyMap<string, string>): Promise<void>;
+    replacePageTemplates(templateKeysByUrl: ReadonlyMap<string, string>, clusterReasonsByTemplateKey?: ReadonlyMap<string, TemplateClusterReason>): Promise<void>;
     /**
      * Promote previously-external pages that now fall under the (possibly extended)
      * scope back to a pending state so that the crawler re-scrapes them as fully
@@ -234,11 +306,46 @@ export default class Archive extends ArchiveAccessor {
      * @returns The URLs of the pages that were reset to pending.
      */
     resetFailedPages(): Promise<string[]>;
+    /**
+     * Persists the raw bytes of an `--inventory` source URL list into the
+     * archive's tar payload, at `inventory/<sha256>.txt`.
+     *
+     * The file name is the content hash rather than the original file name:
+     * re-applying the same list is then a no-op write (`fs.writeFile`
+     * overwrites identical bytes), and the original name — which may embed a
+     * client/project identifier — is never retained (the archive already
+     * omits the source file's absolute path for the same reason; see
+     * `CrawlerOrchestrator.inventory`'s `source` param).
+     *
+     * This bypasses the namespace-scoped {@link ArchiveAccessor.setData} API
+     * (that one is reserved for analyze plugins and requires a namespace) —
+     * this always lands under the fixed `inventory/` prefix regardless of
+     * how this accessor was constructed. Callers that need to read the
+     * saved list back can use the inherited `getData(`inventory/${sha256}`,
+     * 'txt')`, since it resolves to the same path when no namespace is set.
+     *
+     * No entry is ever removed here — same accepted gap as `page_html_blobs`
+     * (a future #23 GC pass will sweep unreachable hashes across both). A
+     * source list that differs byte-for-byte on every run (e.g. a
+     * regenerated doc-root export with fresh timestamps) adds one entry per
+     * run with no pruning of superseded ones.
+     * @param sha256 - Lower-case hex SHA-256 digest of `bytes` (used as the file name).
+     * @param bytes - The exact bytes of the source list file, written verbatim.
+     */
+    saveInventorySourceList(sha256: string, bytes: Buffer): Promise<void>;
     /**
      * Stores the crawl configuration into the archive database.
      * @param config - The configuration object to store.
      */
     setConfig(config: Config): Promise<number[]>;
+    /**
+     * Replaces one page's captured console messages / page errors in the
+     * archive database.
+     * @param pageUrl - The originally-requested URL, normalised (`withoutHashAndAuth` form).
+     * @param redirectPaths - The redirect chain hops captured during fetch, in order.
+     * @param entries - The console log entries to persist.
+     */
+    setConsoleLogs(pageUrl: string, redirectPaths: readonly string[], entries: readonly ConsoleLogEntry[]): Promise<void>;
     /**
      * Stores an external page's data in the archive database without storing
      * an HTML snapshot. External-page rows carry only metadata (status, title,
@@ -311,10 +418,17 @@ export default class Archive extends ArchiveAccessor {
      *
      * Checkpoints the SQLite WAL so the database is self-contained inside
      * `db.sqlite`, renames the temporary working directory to the archive's
-     * basename, and tars it into the final `.nitpicker`. The tar container
-     * holds a single `db.sqlite` file (the legacy `snapshot-html.zip` is gone
-     * — HTML lives as BLOBs in the DB), so finalisation is effectively a
-     * single-file copy with no per-snapshot syscalls.
+     * basename, and tars the **entire tmpDir**. `db.sqlite` is normally the
+     * only entry (HTML lives as BLOBs in the DB, not a `snapshot-html.zip`),
+     * but a namespace-scoped `setData` write (analyze output) or
+     * `saveInventorySourceList` (a saved `--inventory` source list) adds
+     * extra files under tmpDir that get tarred right alongside it.
+     *
+     * This is why every writer path that reaches `write()` must open with
+     * `openPluginData: true` — `Archive.open`'s default extracts only
+     * `db.sqlite`, so a re-crawl (`append` / `inventory` / `retryFailed`)
+     * opened without it would tar back a tmpDir missing those extra files,
+     * silently dropping them from the rewritten archive.
      */
     write(): Promise<void>;
     /** The file extension for Nitpicker archive files (without the leading dot). */
@@ -482,7 +596,19 @@ type ArchiveOptions = {
  * Additional options for opening an existing archive.
  */
 type ArchiveOpenOptions = {
-    /** When true, extracts all files including plugin data. When false, only extracts the database and snapshots. */
+    /**
+     * When `false` (the default), only `db.sqlite` is extracted into tmpDir.
+     * When `true`, every tar entry is extracted, including non-namespace
+     * files written via {@link ArchiveAccessor.setData} (analyze output) or
+     * {@link Archive.saveInventorySourceList} (a saved `--inventory` source
+     * list).
+     *
+     * Every writer path that later calls {@link Archive.write} MUST pass
+     * `true`: `write()` re-tars whatever is currently in tmpDir, so a
+     * re-crawl (`append` / `inventory` / `retryFailed`) opened with the
+     * default would tar back a tmpDir missing those extra files, silently
+     * dropping them from the rewritten archive.
+     */
     openPluginData?: boolean;
 };
 export {};

package/lib/archive/archive.js

@@ -10,18 +10,23 @@ import { dbLog, log, saveLog } from './debug.js';
 import { appendText } from './filesystem/append-text.js';
 import { exists } from './filesystem/exists.js';
 import { isDir } from './filesystem/is-dir.js';
+import { outputBinary } from './filesystem/output-binary.js';
 import { peekTarTopDir } from './filesystem/peek-tar-top-dir.js';
 import { remove } from './filesystem/remove.js';
 import { rename } from './filesystem/rename.js';
 import { tar } from './filesystem/tar.js';
 import { untar } from './filesystem/untar.js';
+import { safePath } from './safe-path.js';
 /**
  * Main archive class for creating, opening, resuming, and writing Nitpicker
  * archive files (`.nitpicker`).
  *
- * An Archive wraps a single SQLite database into a tar archive. HTML
- * bodies live inside the same DB as zstd-compressed BLOBs (see #75) — the
- * tar payload is effectively just `db.sqlite`. It extends
+ * An Archive wraps a SQLite database into a tar archive. HTML bodies live
+ * inside the same DB as zstd-compressed BLOBs (see #75), so `db.sqlite` is
+ * normally the tar's only entry — but it is not the only entry the format
+ * allows: {@link ArchiveAccessor.setData} (namespace-scoped analyze output)
+ * and {@link Archive.saveInventorySourceList} (a saved `--inventory`
+ * source list) add plain files alongside it. It extends
  * {@link ArchiveAccessor} to provide read access to stored data.
  *
  * Use the static factory methods ({@link Archive.create}, {@link Archive.open},
@@ -89,6 +94,20 @@ export default class Archive extends ArchiveAccessor {
      * @deprecated This method is no longer functional.
      */
     abort() { }
+    /**
+     * Adds onto the `rejected_count` of a shape's `dedupe_cap_events` row,
+     * looked up by `shape_key` rather than `id` — used for a shape that
+     * capped in an earlier session (preloaded into `DedupeCapTracker`'s
+     * sticky set) and so has no event id from the current session.
+     *
+     * Thin facade over {@link Database.accumulateDedupeCapRejectedCount}.
+     * @param shapeKey - The capped shape whose rejection count to accumulate.
+     * @param rejectedCount - Additional anchors rejected for this shape in the current session.
+     */
+    async accumulateDedupeCapRejectedCount(shapeKey, rejectedCount) {
+        dbLog('Accumulate dedupe cap rejected count shapeKey=%s rejectedCount=%d', shapeKey, rejectedCount);
+        return await this.#db.accumulateDedupeCapRejectedCount(shapeKey, rejectedCount);
+    }
     /**
      * Records a crawler-level error to both the human-readable `error.log` (full
      * stack, for debugging) and the structured `crawl_errors` table (queryable,
@@ -119,6 +138,29 @@ export default class Archive extends ArchiveAccessor {
         dbLog('Add page error: %s [%s]', url, phase);
         await this.#db.insertPageError(url, phase, message, isExternal);
     }
+    /**
+     * Closes an outage row by stamping `ended_at` — a no-op if already closed.
+     *
+     * Thin facade over {@link Database.closeNetworkOutage}.
+     * @param id - The `network_outages.id` to close.
+     * @param endedAt - Epoch ms the outage is considered to have ended.
+     */
+    async closeNetworkOutage(id, endedAt) {
+        dbLog('Close network outage id=%d endedAt=%d', id, endedAt);
+        return await this.#db.closeNetworkOutage(id, endedAt);
+    }
+    /**
+     * Finalizes a `dedupe_cap_events` row by stamping `rejected_count` — a
+     * no-op if already finalized.
+     *
+     * Thin facade over {@link Database.finalizeDedupeCapEvent}.
+     * @param id - The `dedupe_cap_events.id` to finalize.
+     * @param rejectedCount - Number of anchors rejected for this shape after it capped.
+     */
+    async finalizeDedupeCapEvent(id, rejectedCount) {
+        dbLog('Finalize dedupe cap event id=%d rejectedCount=%d', id, rejectedCount);
+        return await this.#db.finalizeDedupeCapEvent(id, rejectedCount);
+    }
     /**
      * Retrieves the current crawling state, including lists of scraped and pending URLs.
      * @returns An object with `scraped` and `pending` URL arrays.
@@ -186,6 +228,18 @@ export default class Archive extends ArchiveAccessor {
     async getUrl() {
         return this.#db.getBaseUrl();
     }
+    /**
+     * Appends one row (`rejected_count = NULL`) to the `dedupe_cap_events`
+     * journal.
+     *
+     * Thin facade over {@link Database.insertDedupeCapEvent}.
+     * @param params - The newly-capped shape's fields to record.
+     * @returns The autoincremented `id` of the inserted row.
+     */
+    async insertDedupeCapEvent(params) {
+        dbLog('Insert dedupe cap event: shapeKey=%s', params.shapeKey);
+        return await this.#db.insertDedupeCapEvent(params);
+    }
     /**
      * Pre-insert inventory non-HTML URLs as `source='inventory-seed'`
      * placeholders in the `resources` table — the non-HTML counterpart of
@@ -227,6 +281,29 @@ export default class Archive extends ArchiveAccessor {
         dbLog('Insert inventory seeds: %d URL(s)', urls.length);
         await this.#db.insertInventorySeeds(urls.map((u) => u.withoutHashAndAuth));
     }
+    /**
+     * Appends one open row to the `network_outages` journal.
+     *
+     * Thin facade over {@link Database.insertNetworkOutage} — see
+     * {@link recordInventoryRun}'s docstring for why this indirection exists.
+     * @param params - The confirmed-outage fields to record.
+     * @returns The autoincremented `id` of the inserted row.
+     */
+    async insertNetworkOutage(params) {
+        dbLog('Insert network outage: startedAt=%d probeHost=%s', params.startedAt, params.probeHost);
+        return await this.#db.insertNetworkOutage(params);
+    }
+    /**
+     * Every distinct `dedupe_cap_events.shape_key` recorded in this archive.
+     * Consumed by `CrawlerOrchestrator` to preload `DedupeCapTracker`'s
+     * sticky set on `--resume` / `--append` / `--retry-failed` /
+     * `--inventory`, mirroring {@link listDnsBurnedHostCandidates}'s
+     * writer-only exposure.
+     * @returns Distinct shape keys already confirmed capped.
+     */
+    async listDedupeCapShapeKeys() {
+        return this.#db.listDedupeCapShapeKeys();
+    }
     /**
      * Hostnames whose `crawl_errors` history is consistently DNS failures and
      * for which no recent 2xx/3xx page or resource is recorded. Consumed by
@@ -243,6 +320,15 @@ export default class Archive extends ArchiveAccessor {
     async listDnsBurnedHostCandidates() {
         return this.#db.listDnsBurnedHostCandidates();
     }
+    /**
+     * Lists every recorded outage as a resolved {@link OutageWindow}.
+     *
+     * Thin facade over {@link Database.listNetworkOutages}.
+     * @returns Resolved outage windows, or `[]` if none have been recorded.
+     */
+    async listNetworkOutages() {
+        return await this.#db.listNetworkOutages();
+    }
     /**
      * Appends one row to the `inventory_runs` audit log.
      *
@@ -295,9 +381,13 @@ export default class Archive extends ArchiveAccessor {
      * reaching into the low-level database class directly.
      * @param templateKeysByUrl - Page URL → template key, as produced by
      *   `@nitpicker/core`'s `classifyPageTemplates`.
+     * @param clusterReasonsByTemplateKey - Template key → cluster-selection
+     *   evidence, if the caller captured it. Omitting this always clears the
+     *   previously-stored reasons too — "no reason" means "not captured for
+     *   this run", never "carry over the previous run's reasons".
      */
-    async replacePageTemplates(templateKeysByUrl) {
-        await this.#db.replacePageTemplates(templateKeysByUrl);
+    async replacePageTemplates(templateKeysByUrl, clusterReasonsByTemplateKey) {
+        await this.#db.replacePageTemplates(templateKeysByUrl, clusterReasonsByTemplateKey);
     }
     /**
      * Promote previously-external pages that now fall under the (possibly extended)
@@ -322,6 +412,36 @@ export default class Archive extends ArchiveAccessor {
         dbLog('Reset failed pages back to pending');
         return this.#db.resetFailedPages();
     }
+    /**
+     * Persists the raw bytes of an `--inventory` source URL list into the
+     * archive's tar payload, at `inventory/<sha256>.txt`.
+     *
+     * The file name is the content hash rather than the original file name:
+     * re-applying the same list is then a no-op write (`fs.writeFile`
+     * overwrites identical bytes), and the original name — which may embed a
+     * client/project identifier — is never retained (the archive already
+     * omits the source file's absolute path for the same reason; see
+     * `CrawlerOrchestrator.inventory`'s `source` param).
+     *
+     * This bypasses the namespace-scoped {@link ArchiveAccessor.setData} API
+     * (that one is reserved for analyze plugins and requires a namespace) —
+     * this always lands under the fixed `inventory/` prefix regardless of
+     * how this accessor was constructed. Callers that need to read the
+     * saved list back can use the inherited `getData(`inventory/${sha256}`,
+     * 'txt')`, since it resolves to the same path when no namespace is set.
+     *
+     * No entry is ever removed here — same accepted gap as `page_html_blobs`
+     * (a future #23 GC pass will sweep unreachable hashes across both). A
+     * source list that differs byte-for-byte on every run (e.g. a
+     * regenerated doc-root export with fresh timestamps) adds one entry per
+     * run with no pruning of superseded ones.
+     * @param sha256 - Lower-case hex SHA-256 digest of `bytes` (used as the file name).
+     * @param bytes - The exact bytes of the source list file, written verbatim.
+     */
+    async saveInventorySourceList(sha256, bytes) {
+        const filePath = safePath(this.tmpDir, 'inventory', `${sha256}.txt`);
+        await outputBinary(filePath, bytes);
+    }
     /**
      * Stores the crawl configuration into the archive database.
      * @param config - The configuration object to store.
@@ -330,6 +450,17 @@ export default class Archive extends ArchiveAccessor {
         dbLog('Set config: %O', config);
         return this.#db.setConfig(config);
     }
+    /**
+     * Replaces one page's captured console messages / page errors in the
+     * archive database.
+     * @param pageUrl - The originally-requested URL, normalised (`withoutHashAndAuth` form).
+     * @param redirectPaths - The redirect chain hops captured during fetch, in order.
+     * @param entries - The console log entries to persist.
+     */
+    async setConsoleLogs(pageUrl, redirectPaths, entries) {
+        dbLog('Set console logs: %d entries on %s', entries.length, pageUrl);
+        await this.#db.replaceConsoleLogs(pageUrl, redirectPaths, entries);
+    }
     /**
      * Stores an external page's data in the archive database without storing
      * an HTML snapshot. External-page rows carry only metadata (status, title,
@@ -423,10 +554,17 @@ export default class Archive extends ArchiveAccessor {
      *
      * Checkpoints the SQLite WAL so the database is self-contained inside
      * `db.sqlite`, renames the temporary working directory to the archive's
-     * basename, and tars it into the final `.nitpicker`. The tar container
-     * holds a single `db.sqlite` file (the legacy `snapshot-html.zip` is gone
-     * — HTML lives as BLOBs in the DB), so finalisation is effectively a
-     * single-file copy with no per-snapshot syscalls.
+     * basename, and tars the **entire tmpDir**. `db.sqlite` is normally the
+     * only entry (HTML lives as BLOBs in the DB, not a `snapshot-html.zip`),
+     * but a namespace-scoped `setData` write (analyze output) or
+     * `saveInventorySourceList` (a saved `--inventory` source list) adds
+     * extra files under tmpDir that get tarred right alongside it.
+     *
+     * This is why every writer path that reaches `write()` must open with
+     * `openPluginData: true` — `Archive.open`'s default extracts only
+     * `db.sqlite`, so a re-crawl (`append` / `inventory` / `retryFailed`)
+     * opened without it would tar back a tmpDir missing those extra files,
+     * silently dropping them from the rewritten archive.
      */
     async write() {
         saveLog('Starts: %s', this.#filePath);

package/lib/archive/body-hash/compute-body-hash.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Computes a content hash of a page's `<body>`, after normalizing away the
+ * kinds of incidental variance that would otherwise make two structurally
+ * identical pages hash differently: `/index.{ext}` URL-suffix forms and
+ * embedded dynamic tokens (cache-busting hashes, session/order ids, per-build
+ * CSS-module suffixes).
+ *
+ * Only the resulting hash is persisted (`page_meta.body_hash`) — the masked
+ * intermediate string is never stored. The unmasked original HTML remains
+ * fully recoverable from `page_html_blobs`, so nothing is lost by discarding
+ * it here.
+ * @param html - A full HTML document string (or fragment).
+ * @returns 32-byte SHA-256 hash of the masked `<body>` content, ready to
+ *   insert into a `BLOB` column.
+ * @example
+ * ```ts
+ * const hashA = computeBodyHash('<body><a href="/p/a1b2c3d4">x</a></body>');
+ * const hashB = computeBodyHash('<body><a href="/p/z9y8x7w6">x</a></body>');
+ * hashA.equals(hashB); // true — the differing token is masked before hashing
+ * ```
+ */
+export declare function computeBodyHash(html: string): Buffer;

package/lib/archive/body-hash/compute-body-hash.js

@@ -0,0 +1,31 @@
+import { computeContentHash } from '../populate-ref-tables/compute-content-hash.js';
+import { extractBody } from './extract-body.js';
+import { maskDynamicIds } from './mask-dynamic-ids.js';
+import { normalizeUrlLikeStrings } from './normalize-url-like-strings.js';
+/**
+ * Computes a content hash of a page's `<body>`, after normalizing away the
+ * kinds of incidental variance that would otherwise make two structurally
+ * identical pages hash differently: `/index.{ext}` URL-suffix forms and
+ * embedded dynamic tokens (cache-busting hashes, session/order ids, per-build
+ * CSS-module suffixes).
+ *
+ * Only the resulting hash is persisted (`page_meta.body_hash`) — the masked
+ * intermediate string is never stored. The unmasked original HTML remains
+ * fully recoverable from `page_html_blobs`, so nothing is lost by discarding
+ * it here.
+ * @param html - A full HTML document string (or fragment).
+ * @returns 32-byte SHA-256 hash of the masked `<body>` content, ready to
+ *   insert into a `BLOB` column.
+ * @example
+ * ```ts
+ * const hashA = computeBodyHash('<body><a href="/p/a1b2c3d4">x</a></body>');
+ * const hashB = computeBodyHash('<body><a href="/p/z9y8x7w6">x</a></body>');
+ * hashA.equals(hashB); // true — the differing token is masked before hashing
+ * ```
+ */
+export function computeBodyHash(html) {
+    const body = extractBody(html);
+    const normalized = normalizeUrlLikeStrings(body);
+    const masked = maskDynamicIds(normalized);
+    return computeContentHash(masked);
+}

package/lib/archive/body-hash/extract-body.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Extracts the inner HTML of the first `<body>` element from a full HTML
+ * document string.
+ *
+ * Uses a greedy match (`[\s\S]*`, not `[\s\S]*?`) so a literal `<body>`
+ * substring appearing inside the real body (e.g. an inline code sample) does
+ * not truncate the extracted content at that inner occurrence — the match
+ * always extends to the last `</body>` in the document.
+ *
+ * Falls back to returning the full input unchanged when no `<body>` tag is
+ * found (fragment HTML, a page that failed to render, or a snapshot cut off
+ * mid-render) rather than throwing, so callers never need a separate
+ * not-found branch.
+ * @param html - A full HTML document string, or a fragment.
+ * @returns The content between `<body...>` and `</body>`, or `html` unchanged
+ *   if no `<body>` tag is present.
+ */
+export declare function extractBody(html: string): string;

package/lib/archive/body-hash/extract-body.js

@@ -0,0 +1,29 @@
+// The opening tag's attribute span is matched attribute-aware
+// (`(?:"[^"]*"|'[^']*'|[^"'>])*`, not a plain `[^>]*`) so a literal `>`
+// inside a quoted attribute value (e.g. `<body data-x="a>b">`) — legal HTML,
+// since only the delimiting quote character itself must not appear
+// unescaped inside the value — does not end the match early. A plain
+// `[^>]*` would stop at that inner `>`, so the captured body would start
+// mid-attribute instead of at the real content.
+const BODY_PATTERN = /<body(?:"[^"]*"|'[^']*'|[^"'>])*>([\s\S]*)<\/body>/i;
+/**
+ * Extracts the inner HTML of the first `<body>` element from a full HTML
+ * document string.
+ *
+ * Uses a greedy match (`[\s\S]*`, not `[\s\S]*?`) so a literal `<body>`
+ * substring appearing inside the real body (e.g. an inline code sample) does
+ * not truncate the extracted content at that inner occurrence — the match
+ * always extends to the last `</body>` in the document.
+ *
+ * Falls back to returning the full input unchanged when no `<body>` tag is
+ * found (fragment HTML, a page that failed to render, or a snapshot cut off
+ * mid-render) rather than throwing, so callers never need a separate
+ * not-found branch.
+ * @param html - A full HTML document string, or a fragment.
+ * @returns The content between `<body...>` and `</body>`, or `html` unchanged
+ *   if no `<body>` tag is present.
+ */
+export function extractBody(html) {
+    const match = BODY_PATTERN.exec(html);
+    return match?.[1] ?? html;
+}

package/lib/archive/body-hash/mask-dynamic-ids.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Replaces mixed-alphanumeric tokens of 8+ characters with a fixed
+ * placeholder, so that two bodies differing only in an embedded dynamic
+ * value (a cache-busting hash, a per-build CSS-module suffix, a session or
+ * order id) hash the same.
+ *
+ * A token is masked only when it mixes letters and digits — a pure-digit
+ * match (phone numbers, product codes, dates) or a pure-alphabetic match (an
+ * ordinary English word) is left untouched, since those are far more likely
+ * to be stable content than a dynamic identifier.
+ *
+ * This is a heuristic, not a semantic classifier, and it is accepted as such:
+ * a mixed-alphanumeric SKU/order/product code that is itself the only
+ * distinguishing content between two otherwise-identical pages (e.g.
+ * `ABC12345` vs `XYZ98765`) also gets masked, collapsing two genuinely
+ * different pages into one `computeBodyHash` duplicate-detection group — a
+ * false positive. Narrowing the pattern to reduce that risk would also
+ * narrow what it catches (the whole point of this mask is to absorb
+ * cache-busting hashes and CSS-module suffixes whose shape is
+ * indistinguishable from such a code), so this trade-off is deliberate
+ * rather than something a stricter regex would cleanly resolve.
+ *
+ * The placeholder is a fixed constant rather than a per-match incrementing
+ * id: nothing needs to reverse the substitution back to the original value —
+ * the full original HTML is already preserved verbatim in `page_html_blobs`,
+ * so recovering it here would be redundant. All that matters for duplicate
+ * detection is that every masked token collapses to the same value.
+ * @param text - Text to mask (already URL-normalized by
+ *   `normalizeUrlLikeStrings`).
+ * @returns `text` with every mixed-alphanumeric 8+ character token replaced
+ *   by a fixed placeholder.
+ */
+export declare function maskDynamicIds(text: string): string;

package/lib/archive/body-hash/mask-dynamic-ids.js

@@ -0,0 +1,44 @@
+const DYNAMIC_ID_PATTERN = /[a-z0-9]{8,}/gi;
+const DIGITS_ONLY_PATTERN = /^\d+$/;
+const ALPHA_ONLY_PATTERN = /^[a-z]+$/i;
+const MASK_PLACEHOLDER = '__MASKED_ID__';
+/**
+ * Replaces mixed-alphanumeric tokens of 8+ characters with a fixed
+ * placeholder, so that two bodies differing only in an embedded dynamic
+ * value (a cache-busting hash, a per-build CSS-module suffix, a session or
+ * order id) hash the same.
+ *
+ * A token is masked only when it mixes letters and digits — a pure-digit
+ * match (phone numbers, product codes, dates) or a pure-alphabetic match (an
+ * ordinary English word) is left untouched, since those are far more likely
+ * to be stable content than a dynamic identifier.
+ *
+ * This is a heuristic, not a semantic classifier, and it is accepted as such:
+ * a mixed-alphanumeric SKU/order/product code that is itself the only
+ * distinguishing content between two otherwise-identical pages (e.g.
+ * `ABC12345` vs `XYZ98765`) also gets masked, collapsing two genuinely
+ * different pages into one `computeBodyHash` duplicate-detection group — a
+ * false positive. Narrowing the pattern to reduce that risk would also
+ * narrow what it catches (the whole point of this mask is to absorb
+ * cache-busting hashes and CSS-module suffixes whose shape is
+ * indistinguishable from such a code), so this trade-off is deliberate
+ * rather than something a stricter regex would cleanly resolve.
+ *
+ * The placeholder is a fixed constant rather than a per-match incrementing
+ * id: nothing needs to reverse the substitution back to the original value —
+ * the full original HTML is already preserved verbatim in `page_html_blobs`,
+ * so recovering it here would be redundant. All that matters for duplicate
+ * detection is that every masked token collapses to the same value.
+ * @param text - Text to mask (already URL-normalized by
+ *   `normalizeUrlLikeStrings`).
+ * @returns `text` with every mixed-alphanumeric 8+ character token replaced
+ *   by a fixed placeholder.
+ */
+export function maskDynamicIds(text) {
+    return text.replaceAll(DYNAMIC_ID_PATTERN, (match) => {
+        if (DIGITS_ONLY_PATTERN.test(match) || ALPHA_ONLY_PATTERN.test(match)) {
+            return match;
+        }
+        return MASK_PLACEHOLDER;
+    });
+}

package/lib/archive/body-hash/normalize-url-like-strings.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Normalizes `/index.{ext}` path suffixes (`/index.html`, `/index.php`, ...)
+ * to a bare trailing `/` throughout the given text, so that two otherwise
+ * identical bodies that differ only in which equivalent URL form a template
+ * happened to render (`/about/` vs `/about/index.html`) hash the same.
+ *
+ * Applied as a blanket string sweep over the whole body — not scoped to
+ * `href`/`src` attribute values — because the same trailing-suffix variance
+ * can appear anywhere a URL-shaped string is rendered as text (breadcrumbs,
+ * "print this page" widgets, JSON embedded in an inline `<script>`), and the
+ * body is treated as an opaque string for masking purposes (see
+ * `computeBodyHash`).
+ * @param body - The `<body>` inner HTML (or any text) to normalize.
+ * @returns `body` with every `/index.{ext}` suffix collapsed to `/`.
+ */
+export declare function normalizeUrlLikeStrings(body: string): string;