@nitpicker/crawler 0.14.0 → 0.16.0

package/lib/archive/database.d.ts

@@ -1,6 +1,9 @@
+import type { TemplateClusterReason } from './db-ops/analysis/types.js';
 import type { JsonLdRow, MainContentAudioRow, MainContentButtonRow, MainContentCanvasRow, MainContentHeadingRow, MainContentIframeRow, MainContentImageRow, MainContentTableRow, MainContentVideoRow, TagRow } from './meta/types.js';
-import type { Config, DatabaseOption, DB_Redirect, DB_Resource, DatabaseEvent, InventoryRunMeta, PageFilter, PageSource } from './types.js';
+import type { Config, DatabaseOption, DB_Redirect, DB_Resource, DatabaseEvent, InsertDedupeCapEventParams, InsertNetworkOutageParams, InventoryRunMeta, PageFilter, PageSource } from './types.js';
+import type { OutageWindow } from '../is-within-outage-window.js';
 import type { PageData, Resource } from '../utils/types/types.js';
+import type { ConsoleLogEntry } from '@d-zero/beholder';
 import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
 import type { Knex } from 'knex';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
@@ -29,16 +32,39 @@ import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-awa
 export declare class Database extends EventEmitter<DatabaseEvent> {
     #private;
     private constructor();
+    /**
+     * Adds onto the `rejected_count` of a shape's `dedupe_cap_events` row,
+     * looked up by `shape_key` rather than `id`. Delegates to
+     * {@link accumulateDedupeCapRejectedCountOp}.
+     * @param shapeKey - The capped shape whose rejection count to accumulate.
+     * @param rejectedCount - Additional anchors rejected for this shape in the current session.
+     */
+    accumulateDedupeCapRejectedCount(shapeKey: string, rejectedCount: number): Promise<void>;
     /**
      * Forces a WAL checkpoint, writing all pending WAL data back to the main
      * database file. Delegates to {@link checkpointOp}.
      */
     checkpoint(): Promise<void>;
+    /**
+     * Closes an outage row by stamping `ended_at` — a no-op if the row is
+     * already closed. Delegates to {@link closeNetworkOutageOp}.
+     * @param id - The `network_outages.id` to close.
+     * @param endedAt - Epoch ms the outage is considered to have ended.
+     */
+    closeNetworkOutage(id: number, endedAt: number): Promise<void>;
     /**
      * Destroys the database connection, releasing all pooled resources.
      * Delegates to {@link destroyOp}.
      */
     destroy(): Promise<void>;
+    /**
+     * Finalizes a `dedupe_cap_events` row by stamping `rejected_count` — a
+     * no-op if the row is already finalized. Delegates to
+     * {@link finalizeDedupeCapEventOp}.
+     * @param id - The `dedupe_cap_events.id` to finalize.
+     * @param rejectedCount - Number of anchors rejected for this shape after it capped.
+     */
+    finalizeDedupeCapEvent(id: number, rejectedCount: number): Promise<void>;
     /**
      * Retrieves all anchors (outgoing links) on a specific page.
      * Delegates to {@link getAnchorsOnPageOp}.
@@ -263,6 +289,13 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @param isExternal - Whether the URL is external to the crawl scope.
      */
     insertCrawlError(url: string | null, message: string, isExternal?: boolean): Promise<void>;
+    /**
+     * Appends one row (`rejected_count = NULL`) to the `dedupe_cap_events`
+     * journal. Delegates to {@link insertDedupeCapEventOp}.
+     * @param params - The newly-capped shape's fields to record.
+     * @returns The autoincremented `id` of the newly-inserted row.
+     */
+    insertDedupeCapEvent(params: InsertDedupeCapEventParams): Promise<number>;
     /**
      * Pre-insert inventory non-HTML URLs into `resources` as placeholder rows.
      * Delegates to {@link insertInventoryResourcesOp}.
@@ -277,6 +310,13 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @param urls - URL strings already in `withoutHashAndAuth` form.
      */
     insertInventorySeeds(urls: readonly string[]): Promise<void>;
+    /**
+     * Appends one open (`ended_at = NULL`) row to the `network_outages`
+     * journal. Delegates to {@link insertNetworkOutageOp}.
+     * @param params - The confirmed-outage fields to record.
+     * @returns The autoincremented `id` of the newly-inserted row.
+     */
+    insertNetworkOutage(params: InsertNetworkOutageParams): Promise<number>;
     /**
      * Records a partial scrape failure against the page identified by `url`.
      * Delegates to {@link insertPageErrorOp}.
@@ -300,6 +340,13 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @param pageUrl - The URL of the page that references the resource.
      */
     insertResourceReferrers(src: string, pageUrl: string): Promise<void>;
+    /**
+     * Every distinct `dedupe_cap_events.shape_key` recorded in this archive.
+     * Delegates to {@link listDedupeCapShapeKeysOp}.
+     * @returns Distinct shape keys already confirmed capped, or `[]` on an
+     *   archive that predates `dedupe_cap_events` or has recorded none.
+     */
+    listDedupeCapShapeKeys(): Promise<string[]>;
     /**
      * Hostnames whose `crawl_errors` history is consistently DNS failures and
      * for which no recent 2xx-3xx page or resource is recorded.
@@ -307,6 +354,13 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @returns Lower-cased hostnames safe to short-circuit.
      */
     listDnsBurnedHostCandidates(): Promise<string[]>;
+    /**
+     * Lists every recorded outage as a resolved {@link OutageWindow}.
+     * Delegates to {@link listNetworkOutagesOp}.
+     * @returns Resolved outage windows, or `[]` on an archive that predates
+     *   `network_outages` or has recorded no outages.
+     */
+    listNetworkOutages(): Promise<OutageWindow[]>;
     /**
      * Appends one row to the `inventory_runs` audit log.
      * Delegates to {@link recordInventoryRunOp}.
@@ -339,12 +393,23 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
         line?: number | null;
         col?: number | null;
     }[]): Promise<void>;
+    /**
+     * Replaces one page's `page_console_logs` rows with a freshly captured
+     * set of console messages / page errors. Delegates to
+     * {@link replaceConsoleLogsOp}.
+     * @param pageUrl - The originally-requested URL, normalised (`withoutHashAndAuth` form).
+     * @param redirectPaths - The redirect chain hops captured during fetch, in order.
+     * @param entries - The console log entries to persist.
+     */
+    replaceConsoleLogs(pageUrl: string, redirectPaths: readonly string[], entries: readonly ConsoleLogEntry[]): Promise<void>;
     /**
      * Replaces the stored DOM-structure template classification with a
      * freshly generated set. Delegates to {@link replacePageTemplatesOp}.
      * @param templateKeysByUrl - Page URL → template key.
+     * @param clusterReasonsByTemplateKey - Template key → cluster-selection
+     *   evidence, if the caller captured it.
      */
-    replacePageTemplates(templateKeysByUrl: ReadonlyMap<string, string>): Promise<void>;
+    replacePageTemplates(templateKeysByUrl: ReadonlyMap<string, string>, clusterReasonsByTemplateKey?: ReadonlyMap<string, TemplateClusterReason>): Promise<void>;
     /**
      * Promote previously-external pages whose URL falls under any of the new
      * scope entries back to a "needs scraping" state.

package/lib/archive/database.js

@@ -15,6 +15,11 @@ import { getConfig as getConfigOp } from './db-ops/config/get-config.js';
 import { getName as getNameOp } from './db-ops/config/get-name.js';
 import { setConfig as setConfigOp } from './db-ops/config/set-config.js';
 import { updateConfig as updateConfigOp } from './db-ops/config/update-config.js';
+import { replaceConsoleLogs as replaceConsoleLogsOp } from './db-ops/console-logs/replace-console-logs.js';
+import { accumulateDedupeCapRejectedCount as accumulateDedupeCapRejectedCountOp } from './db-ops/dedupe-cap/accumulate-dedupe-cap-rejected-count.js';
+import { finalizeDedupeCapEvent as finalizeDedupeCapEventOp } from './db-ops/dedupe-cap/finalize-dedupe-cap-event.js';
+import { insertDedupeCapEvent as insertDedupeCapEventOp } from './db-ops/dedupe-cap/insert-dedupe-cap-event.js';
+import { listDedupeCapShapeKeys as listDedupeCapShapeKeysOp } from './db-ops/dedupe-cap/list-dedupe-cap-shape-keys.js';
 import { insertCrawlError as insertCrawlErrorOp } from './db-ops/errors/insert-crawl-error.js';
 import { insertPageError as insertPageErrorOp } from './db-ops/errors/insert-page-error.js';
 import { listDnsBurnedHostCandidates as listDnsBurnedHostCandidatesOp } from './db-ops/errors/list-dns-burned-host-candidates.js';
@@ -33,6 +38,9 @@ import { getMainContentImagesOfPage as getMainContentImagesOfPageOp } from './db
 import { getMainContentTablesOfPage as getMainContentTablesOfPageOp } from './db-ops/meta/get-main-content-tables-of-page.js';
 import { getTagsOfPage as getTagsOfPageOp } from './db-ops/meta/get-tags-of-page.js';
 import { getVideosOfPage as getVideosOfPageOp } from './db-ops/meta/get-videos-of-page.js';
+import { closeNetworkOutage as closeNetworkOutageOp } from './db-ops/outages/close-network-outage.js';
+import { insertNetworkOutage as insertNetworkOutageOp } from './db-ops/outages/insert-network-outage.js';
+import { listNetworkOutages as listNetworkOutagesOp } from './db-ops/outages/list-network-outages.js';
 import { setUrlOrder as setUrlOrderOp } from './db-ops/pages/order/set-url-order.js';
 import { getCrawlingState as getCrawlingStateOp } from './db-ops/pages/read/get-crawling-state.js';
 import { getExistingPageUrls as getExistingPageUrlsOp } from './db-ops/pages/read/get-existing-page-urls.js';
@@ -114,6 +122,16 @@ export class Database extends EventEmitter {
             },
         });
     }
+    /**
+     * Adds onto the `rejected_count` of a shape's `dedupe_cap_events` row,
+     * looked up by `shape_key` rather than `id`. Delegates to
+     * {@link accumulateDedupeCapRejectedCountOp}.
+     * @param shapeKey - The capped shape whose rejection count to accumulate.
+     * @param rejectedCount - Additional anchors rejected for this shape in the current session.
+     */
+    async accumulateDedupeCapRejectedCount(shapeKey, rejectedCount) {
+        return emitErrorAndRetry(this, 'Database.accumulateDedupeCapRejectedCount', async () => await accumulateDedupeCapRejectedCountOp(this.#instance, shapeKey, rejectedCount), retrySetting);
+    }
     /**
      * Forces a WAL checkpoint, writing all pending WAL data back to the main
      * database file. Delegates to {@link checkpointOp}.
@@ -121,6 +139,15 @@ export class Database extends EventEmitter {
     async checkpoint() {
         await checkpointOp(this.#instance);
     }
+    /**
+     * Closes an outage row by stamping `ended_at` — a no-op if the row is
+     * already closed. Delegates to {@link closeNetworkOutageOp}.
+     * @param id - The `network_outages.id` to close.
+     * @param endedAt - Epoch ms the outage is considered to have ended.
+     */
+    async closeNetworkOutage(id, endedAt) {
+        return emitErrorAndRetry(this, 'Database.closeNetworkOutage', async () => await closeNetworkOutageOp(this.#instance, id, endedAt), retrySetting);
+    }
     /**
      * Destroys the database connection, releasing all pooled resources.
      * Delegates to {@link destroyOp}.
@@ -128,6 +155,16 @@ export class Database extends EventEmitter {
     async destroy() {
         await destroyOp(this.#instance);
     }
+    /**
+     * Finalizes a `dedupe_cap_events` row by stamping `rejected_count` — a
+     * no-op if the row is already finalized. Delegates to
+     * {@link finalizeDedupeCapEventOp}.
+     * @param id - The `dedupe_cap_events.id` to finalize.
+     * @param rejectedCount - Number of anchors rejected for this shape after it capped.
+     */
+    async finalizeDedupeCapEvent(id, rejectedCount) {
+        return emitErrorAndRetry(this, 'Database.finalizeDedupeCapEvent', async () => await finalizeDedupeCapEventOp(this.#instance, id, rejectedCount), retrySetting);
+    }
     /**
      * Retrieves all anchors (outgoing links) on a specific page.
      * Delegates to {@link getAnchorsOnPageOp}.
@@ -409,6 +446,15 @@ export class Database extends EventEmitter {
     async insertCrawlError(url, message, isExternal = false) {
         return emitErrorAndRetry(this, 'Database.insertCrawlError', async () => await insertCrawlErrorOp(this.#instance, url, message, isExternal), retrySetting);
     }
+    /**
+     * Appends one row (`rejected_count = NULL`) to the `dedupe_cap_events`
+     * journal. Delegates to {@link insertDedupeCapEventOp}.
+     * @param params - The newly-capped shape's fields to record.
+     * @returns The autoincremented `id` of the newly-inserted row.
+     */
+    async insertDedupeCapEvent(params) {
+        return emitErrorAndRetry(this, 'Database.insertDedupeCapEvent', async () => await insertDedupeCapEventOp(this.#instance, params), retrySetting);
+    }
     /**
      * Pre-insert inventory non-HTML URLs into `resources` as placeholder rows.
      * Delegates to {@link insertInventoryResourcesOp}.
@@ -427,6 +473,15 @@ export class Database extends EventEmitter {
     async insertInventorySeeds(urls) {
         return emitErrorAndRetry(this, 'Database.insertInventorySeeds', async () => await insertInventorySeedsOp(this.#instance, this.#writeRefCaches, urls), retrySetting);
     }
+    /**
+     * Appends one open (`ended_at = NULL`) row to the `network_outages`
+     * journal. Delegates to {@link insertNetworkOutageOp}.
+     * @param params - The confirmed-outage fields to record.
+     * @returns The autoincremented `id` of the newly-inserted row.
+     */
+    async insertNetworkOutage(params) {
+        return emitErrorAndRetry(this, 'Database.insertNetworkOutage', async () => await insertNetworkOutageOp(this.#instance, params), retrySetting);
+    }
     /**
      * Records a partial scrape failure against the page identified by `url`.
      * Delegates to {@link insertPageErrorOp}.
@@ -456,6 +511,15 @@ export class Database extends EventEmitter {
     async insertResourceReferrers(src, pageUrl) {
         return emitErrorAndRetry(this, 'Database.insertResourceReferrers', async () => await insertResourceReferrersOp(this.#instance, this.#writeRefCaches, src, pageUrl), retrySetting);
     }
+    /**
+     * Every distinct `dedupe_cap_events.shape_key` recorded in this archive.
+     * Delegates to {@link listDedupeCapShapeKeysOp}.
+     * @returns Distinct shape keys already confirmed capped, or `[]` on an
+     *   archive that predates `dedupe_cap_events` or has recorded none.
+     */
+    async listDedupeCapShapeKeys() {
+        return emitErrorAndRetry(this, 'Database.listDedupeCapShapeKeys', async () => await listDedupeCapShapeKeysOp(this.#instance), retrySetting);
+    }
     /**
      * Hostnames whose `crawl_errors` history is consistently DNS failures and
      * for which no recent 2xx-3xx page or resource is recorded.
@@ -465,6 +529,15 @@ export class Database extends EventEmitter {
     async listDnsBurnedHostCandidates() {
         return emitErrorAndRetry(this, 'Database.listDnsBurnedHostCandidates', async () => await listDnsBurnedHostCandidatesOp(this.#instance), retrySetting);
     }
+    /**
+     * Lists every recorded outage as a resolved {@link OutageWindow}.
+     * Delegates to {@link listNetworkOutagesOp}.
+     * @returns Resolved outage windows, or `[]` on an archive that predates
+     *   `network_outages` or has recorded no outages.
+     */
+    async listNetworkOutages() {
+        return emitErrorAndRetry(this, 'Database.listNetworkOutages', async () => await listNetworkOutagesOp(this.#instance), retrySetting);
+    }
     /**
      * Appends one row to the `inventory_runs` audit log.
      * Delegates to {@link recordInventoryRunOp}.
@@ -494,13 +567,29 @@ export class Database extends EventEmitter {
     async replaceAnalysisViolations(violations) {
         return emitErrorAndRetry(this, 'Database.replaceAnalysisViolations', async () => await replaceAnalysisViolationsOp(this.#instance, violations), retrySetting);
     }
+    /**
+     * Replaces one page's `page_console_logs` rows with a freshly captured
+     * set of console messages / page errors. Delegates to
+     * {@link replaceConsoleLogsOp}.
+     * @param pageUrl - The originally-requested URL, normalised (`withoutHashAndAuth` form).
+     * @param redirectPaths - The redirect chain hops captured during fetch, in order.
+     * @param entries - The console log entries to persist.
+     */
+    async replaceConsoleLogs(pageUrl, redirectPaths, entries) {
+        return emitErrorAndRetry(this, 'Database.replaceConsoleLogs', async () => await replaceConsoleLogsOp(this.#instance, this.#writeRefCaches, pageUrl, redirectPaths, entries), retrySetting);
+    }
     /**
      * Replaces the stored DOM-structure template classification with a
      * freshly generated set. Delegates to {@link replacePageTemplatesOp}.
      * @param templateKeysByUrl - Page URL → template key.
-     */
-    async replacePageTemplates(templateKeysByUrl) {
-        return emitErrorAndRetry(this, 'Database.replacePageTemplates', async () => await replacePageTemplatesOp(this.#instance, templateKeysByUrl), retrySetting);
+     * @param clusterReasonsByTemplateKey - Template key → cluster-selection
+     *   evidence, if the caller captured it.
+     */
+    async replacePageTemplates(templateKeysByUrl, clusterReasonsByTemplateKey) {
+        return emitErrorAndRetry(this, 'Database.replacePageTemplates', async () => await replacePageTemplatesOp(this.#instance, {
+            templateKeysByUrl,
+            clusterReasonsByTemplateKey,
+        }), retrySetting);
     }
     /**
      * Promote previously-external pages whose URL falls under any of the new

package/lib/archive/db-ops/_shared/clear-write-ref-caches.js

@@ -30,5 +30,6 @@ export function clearWriteRefCaches(caches) {
     caches.contentTypeIds.clear();
     caches.jsonIds.clear();
     caches.blobIds.clear();
+    caches.consoleLogIds.clear();
     caches.headers = null;
 }

package/lib/archive/db-ops/_shared/compress-payload.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Compresses raw bytes for storage in a `codec`/`size_raw`/`size_stored`
+ * BLOB column, matching the read side of
+ * {@link import('./decode-json-ref.js').decodeJsonRef} and
+ * {@link import('../../decode-html-blob.js').decodeStoredBlob}.
+ *
+ * The one shared primitive behind every `codec: 'zstd'` writer in the
+ * archive (`upsert-json-ref.ts`, `write-page-html-blob.ts`,
+ * `replace-page-templates.ts`) — a codec change (e.g. a brotli fallback, a
+ * different zstd level) only has to happen here. Callers that also need a
+ * content hash (`upsert-json-ref.ts`, `write-page-html-blob.ts`) compute it
+ * themselves from the same `rawBytes` they pass in here, rather than this
+ * function computing one not every caller needs.
+ * @param rawBytes - The raw (uncompressed) payload bytes.
+ * @returns The compressed body alongside the codec and size bookkeeping the
+ *   BLOB column's schema expects.
+ * @example
+ * const rawBytes = Buffer.from(JSON.stringify(reason), 'utf8');
+ * const { body, codec, sizeRaw, sizeStored } = compressPayload(rawBytes);
+ */
+export declare function compressPayload(rawBytes: Buffer): {
+    readonly body: Buffer;
+    readonly codec: 'zstd';
+    readonly sizeRaw: number;
+    readonly sizeStored: number;
+};

package/lib/archive/db-ops/_shared/compress-payload.js

@@ -0,0 +1,30 @@
+import { zstdCompressSync } from 'node:zlib';
+/**
+ * Compresses raw bytes for storage in a `codec`/`size_raw`/`size_stored`
+ * BLOB column, matching the read side of
+ * {@link import('./decode-json-ref.js').decodeJsonRef} and
+ * {@link import('../../decode-html-blob.js').decodeStoredBlob}.
+ *
+ * The one shared primitive behind every `codec: 'zstd'` writer in the
+ * archive (`upsert-json-ref.ts`, `write-page-html-blob.ts`,
+ * `replace-page-templates.ts`) — a codec change (e.g. a brotli fallback, a
+ * different zstd level) only has to happen here. Callers that also need a
+ * content hash (`upsert-json-ref.ts`, `write-page-html-blob.ts`) compute it
+ * themselves from the same `rawBytes` they pass in here, rather than this
+ * function computing one not every caller needs.
+ * @param rawBytes - The raw (uncompressed) payload bytes.
+ * @returns The compressed body alongside the codec and size bookkeeping the
+ *   BLOB column's schema expects.
+ * @example
+ * const rawBytes = Buffer.from(JSON.stringify(reason), 'utf8');
+ * const { body, codec, sizeRaw, sizeStored } = compressPayload(rawBytes);
+ */
+export function compressPayload(rawBytes) {
+    const compressed = zstdCompressSync(rawBytes);
+    return {
+        body: compressed,
+        codec: 'zstd',
+        sizeRaw: rawBytes.byteLength,
+        sizeStored: compressed.byteLength,
+    };
+}

package/lib/archive/db-ops/_shared/create-write-ref-caches.js

@@ -21,6 +21,7 @@ export function createWriteRefCaches() {
         contentTypeIds: new Map(),
         jsonIds: new Map(),
         blobIds: new Map(),
+        consoleLogIds: new Map(),
         headers: null,
     };
 }

package/lib/archive/db-ops/_shared/types.d.ts

@@ -42,6 +42,8 @@ export interface WriteRefCaches {
     readonly jsonIds: Map<string, number>;
     /** `hex(content hash) → blob_refs.id`. */
     readonly blobIds: Map<string, number>;
+    /** `hex(content hash) → console_log_items.id`. */
+    readonly consoleLogIds: Map<string, number>;
     /**
      * Header dictionary caches (`header_name_refs` / `header_value_refs` /
      * `header_sets` id maps). `null` until the first header-set upsert

package/lib/archive/db-ops/_shared/upsert-json-ref.js

@@ -1,5 +1,5 @@
-import { zstdCompressSync } from 'node:zlib';
 import { computeContentHash } from '../../populate-ref-tables/compute-content-hash.js';
+import { compressPayload } from './compress-payload.js';
 /**
  * Resolves the `json_refs.id` for one JSON payload string, inserting the
  * deduplicated (hash-keyed) row when the payload is new.
@@ -27,11 +27,11 @@ export async function upsertJsonRef(qb, caches, jsonText) {
     if (cached !== undefined) {
         return cached;
     }
-    const compressed = zstdCompressSync(rawBytes);
+    const { body, sizeRaw, sizeStored } = compressPayload(rawBytes);
     const rows = await qb.raw(`INSERT INTO json_refs (hash, json_text, codec, size_raw, size_stored)
 		 VALUES (?, ?, 'zstd', ?, ?)
 		 ON CONFLICT(hash) DO UPDATE SET hash = hash
-		 RETURNING id`, [hash, compressed, rawBytes.byteLength, compressed.byteLength]);
+		 RETURNING id`, [hash, body, sizeRaw, sizeStored]);
     const first = rows[0];
     if (first === undefined) {
         throw new Error('upsertJsonRef: RETURNING yielded no row');

package/lib/archive/db-ops/analysis/replace-page-templates.d.ts

@@ -1,3 +1,4 @@
+import type { ReplacePageTemplatesParams } from './types.js';
 import type { Knex } from 'knex';
 /**
  * Replaces the stored DOM-structure template classification (`--templates`)
@@ -13,8 +14,16 @@ import type { Knex } from 'knex';
  * (e.g. a URL-normalization mismatch between the in-memory `Page.url.href`
  * and the stored `url_refs.url`) should not discard the rest of a
  * potentially multi-thousand-page classification run.
+ *
+ * `page_template_clusters` is always cleared alongside `page_templates`
+ * regardless of whether `clusterReasonsByTemplateKey` is passed — "no
+ * reason" must mean "not captured for this run", never "carry over the
+ * previous run's reason". Reason rows are inserted for every key in
+ * `clusterReasonsByTemplateKey` even if some have no surviving member page
+ * in `templateKeysByUrl` after URL-resolution skips above — harmless
+ * (nothing joins `page_template_clusters` back to `page_templates` by FK;
+ * see the table's own JSDoc), and simpler than cross-filtering the two maps.
  * @param knex - Knex query builder connected to the archive DB.
- * @param templateKeysByUrl - Page URL → template key, as produced by
- *   `classifyPageTemplates`.
+ * @param params - See {@link ReplacePageTemplatesParams}.
  */
-export declare function replacePageTemplates(knex: Knex, templateKeysByUrl: ReadonlyMap<string, string>): Promise<void>;
+export declare function replacePageTemplates(knex: Knex, params: ReplacePageTemplatesParams): Promise<void>;

package/lib/archive/db-ops/analysis/replace-page-templates.js

@@ -1,4 +1,5 @@
 import { eachSplitted } from '../../../utils/array/each-splitted.js';
+import { compressPayload } from '../_shared/compress-payload.js';
 /**
  * Replaces the stored DOM-structure template classification (`--templates`)
  * with a freshly generated set.
@@ -13,13 +14,44 @@ import { eachSplitted } from '../../../utils/array/each-splitted.js';
  * (e.g. a URL-normalization mismatch between the in-memory `Page.url.href`
  * and the stored `url_refs.url`) should not discard the rest of a
  * potentially multi-thousand-page classification run.
+ *
+ * `page_template_clusters` is always cleared alongside `page_templates`
+ * regardless of whether `clusterReasonsByTemplateKey` is passed — "no
+ * reason" must mean "not captured for this run", never "carry over the
+ * previous run's reason". Reason rows are inserted for every key in
+ * `clusterReasonsByTemplateKey` even if some have no surviving member page
+ * in `templateKeysByUrl` after URL-resolution skips above — harmless
+ * (nothing joins `page_template_clusters` back to `page_templates` by FK;
+ * see the table's own JSDoc), and simpler than cross-filtering the two maps.
  * @param knex - Knex query builder connected to the archive DB.
- * @param templateKeysByUrl - Page URL → template key, as produced by
- *   `classifyPageTemplates`.
+ * @param params - See {@link ReplacePageTemplatesParams}.
  */
-export async function replacePageTemplates(knex, templateKeysByUrl) {
+export async function replacePageTemplates(knex, params) {
+    const { templateKeysByUrl, clusterReasonsByTemplateKey } = params;
+    // Compressing every reason is pure CPU work independent of the DB — done
+    // before opening the transaction below so it doesn't extend how long the
+    // SQLite write-lock is held for.
+    const reasonRows = clusterReasonsByTemplateKey && clusterReasonsByTemplateKey.size > 0
+        ? [...clusterReasonsByTemplateKey].map(([templateKey, reason]) => {
+            const { body, codec, sizeRaw, sizeStored } = compressPayload(Buffer.from(JSON.stringify(reason), 'utf8'));
+            return {
+                template_key: templateKey,
+                member_count: reason.memberCount,
+                reason_json: body,
+                codec,
+                size_raw: sizeRaw,
+                size_stored: sizeStored,
+            };
+        })
+        : [];
     await knex.transaction(async (trx) => {
         await trx('page_templates').delete();
+        await trx('page_template_clusters').delete();
+        if (reasonRows.length > 0) {
+            await eachSplitted(reasonRows, 100, async (chunk) => {
+                await trx('page_template_clusters').insert(chunk);
+            });
+        }
         if (templateKeysByUrl.size === 0) {
             return;
         }

package/lib/archive/db-ops/analysis/types.d.ts

@@ -0,0 +1,61 @@
+/**
+ * The six landmark types `@d-zero/page-cluster`'s `extractLandmarks` and
+ * `ClusterReason.landmarks` recognize.
+ */
+export type TemplateClusterLandmarkType = 'header' | 'footer' | 'nav' | 'aside' | 'form' | 'search';
+/**
+ * Why a Pass-0 block (one of possibly several that merged into a final
+ * cluster) was formed — mirrors `@d-zero/page-cluster`'s `BlockingReason`
+ * discriminated union.
+ */
+export type TemplateClusterBlockingReason = {
+    readonly kind: 'css';
+    readonly distinctiveStylesheetHrefs: readonly string[];
+} | {
+    readonly kind: 'path';
+    readonly pathKey: string;
+} | {
+    readonly kind: 'orphanMerge';
+    readonly pathKey: string;
+};
+/** One block's blocking key and the reason it was formed. */
+export interface TemplateClusterBlockingEvidence {
+    readonly blockKey: string;
+    readonly reason: TemplateClusterBlockingReason;
+}
+/** How common one landmark type is across a cluster's member pages. */
+export interface TemplateClusterLandmarkProfile {
+    readonly presenceRate: number;
+    readonly chromeRate: number;
+    readonly shellTokens: readonly string[];
+    readonly memberCountWithInstance: number;
+}
+/**
+ * nitpicker's own copy of `@d-zero/page-cluster`'s `ClusterReason` shape.
+ * Kept independent of the `@d-zero/page-cluster` package (rather than
+ * importing its type directly) so `@nitpicker/query` — which does not
+ * depend on `@d-zero/page-cluster` and is consumed by the browser-side
+ * viewer build — never has to add that dependency just to type a value
+ * read back out of the archive. `@d-zero/page-cluster`'s `ClusterReason`
+ * is structurally assignable to this type.
+ */
+export interface TemplateClusterReason {
+    readonly memberCount: number;
+    readonly blocking: readonly TemplateClusterBlockingEvidence[];
+    readonly structuralCoreTokens: readonly string[];
+    readonly landmarks: Partial<Record<TemplateClusterLandmarkType, TemplateClusterLandmarkProfile>>;
+    readonly siblingClusterKeys: readonly string[];
+}
+/**
+ * Params for {@link import('./replace-page-templates.js').replacePageTemplates}.
+ */
+export interface ReplacePageTemplatesParams {
+    /** Page URL → template key, as produced by `classifyPageTemplates`. */
+    readonly templateKeysByUrl: ReadonlyMap<string, string>;
+    /**
+     * Template key → `@d-zero/page-cluster`'s cluster-selection evidence for
+     * that key, as produced by `classifyPageTemplates`. Omitted entirely (not
+     * just empty) when the caller didn't request reasons.
+     */
+    readonly clusterReasonsByTemplateKey?: ReadonlyMap<string, TemplateClusterReason>;
+}

package/lib/archive/db-ops/analysis/types.js

@@ -0,0 +1 @@
+export {};

package/lib/archive/db-ops/console-logs/compute-console-log-hash.d.ts

@@ -0,0 +1,19 @@
+import type { ConsoleLogContent } from './types.js';
+/**
+ * Computes the `console_log_items.hash` for one console log entry's content.
+ *
+ * Hashing over the raw content fields (not their resolved ref ids) keeps the
+ * dictionary row's identity independent of ref-table state, matching
+ * `text_refs` / `json_refs` / `url_refs`, whose hash columns are likewise
+ * computed from source content rather than downstream ids. The same
+ * message logged by a shared framework on every page therefore hashes to
+ * the same value regardless of insertion order or which page's scrape
+ * resolves the dictionary row first.
+ * @param entry - The content fields to hash. `argsJson` must already be
+ *   `stringifyConsoleLogArgs`'s output — this function does not
+ *   re-serialize `args` itself.
+ * @returns 32-byte SHA-256 hash, ready for the `hash BLOB UNIQUE` column.
+ * @example
+ * const hash = computeConsoleLogHash({ type: 'error', text: 'boom', argsJson: null });
+ */
+export declare function computeConsoleLogHash(entry: ConsoleLogContent): Buffer;

package/lib/archive/db-ops/console-logs/compute-console-log-hash.js

@@ -0,0 +1,30 @@
+import { computeContentHash } from '../../populate-ref-tables/compute-content-hash.js';
+/**
+ * Computes the `console_log_items.hash` for one console log entry's content.
+ *
+ * Hashing over the raw content fields (not their resolved ref ids) keeps the
+ * dictionary row's identity independent of ref-table state, matching
+ * `text_refs` / `json_refs` / `url_refs`, whose hash columns are likewise
+ * computed from source content rather than downstream ids. The same
+ * message logged by a shared framework on every page therefore hashes to
+ * the same value regardless of insertion order or which page's scrape
+ * resolves the dictionary row first.
+ * @param entry - The content fields to hash. `argsJson` must already be
+ *   `stringifyConsoleLogArgs`'s output — this function does not
+ *   re-serialize `args` itself.
+ * @returns 32-byte SHA-256 hash, ready for the `hash BLOB UNIQUE` column.
+ * @example
+ * const hash = computeConsoleLogHash({ type: 'error', text: 'boom', argsJson: null });
+ */
+export function computeConsoleLogHash(entry) {
+    const canonical = JSON.stringify([
+        entry.type,
+        entry.text,
+        entry.argsJson,
+        entry.location?.url ?? null,
+        entry.location?.lineNumber ?? null,
+        entry.location?.columnNumber ?? null,
+        entry.stack ?? null,
+    ]);
+    return computeContentHash(canonical);
+}

package/lib/archive/db-ops/console-logs/replace-console-logs.d.ts

@@ -0,0 +1,34 @@
+import type { WriteRefCaches } from '../_shared/types.js';
+import type { ConsoleLogEntry } from '@d-zero/beholder';
+import type { Knex } from 'knex';
+/**
+ * Replaces one page's `page_console_logs` rows wholesale with a freshly
+ * captured set of console messages / page errors (issue #228).
+ *
+ * Scoped-Replace, the same pattern as `replaceAnchorEdges` /
+ * `replaceImageItems`: the page's existing rows are deleted and the new
+ * set is inserted in the same transaction, so a re-scrape (`--append` /
+ * `--retry-failed` / re-render) never accumulates duplicate occurrences.
+ * Unlike those two, there is no non-empty guard on the DELETE here — the
+ * caller (`Crawler#handleConsoleLogs`) already skips emitting the event
+ * entirely when `entries` is empty, which is what keeps a degraded
+ * re-scrape from wiping out a prior good capture.
+ *
+ * `pageUrl` is the originally-requested URL (matching `updatePage`'s
+ * `page.url.withoutHashAndAuth` contract), not necessarily the page that
+ * ends up holding the content: `resolveRedirectChain` derives the same
+ * redirect destination `updatePage` writes content under, so console logs
+ * attach to the row that actually carries the page's `page_meta`, not to
+ * an empty placeholder for the pre-redirect URL.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param pageUrl - The originally-requested URL, normalised
+ *   (`withoutHashAndAuth` form).
+ * @param redirectPaths - The redirect chain hops captured during fetch, in
+ *   order (empty when the page was not redirected).
+ * @param entries - The console log entries to persist. Must be non-empty —
+ *   callers should skip calling this function entirely for an empty list.
+ * @example
+ * await replaceConsoleLogs(knex, caches, page.url.withoutHashAndAuth, page.redirectPaths, entries);
+ */
+export declare function replaceConsoleLogs(knex: Knex, caches: WriteRefCaches, pageUrl: string, redirectPaths: readonly string[], entries: readonly ConsoleLogEntry[]): Promise<void>;