@nitpicker/crawler 0.13.0 → 0.15.0

package/lib/archive/database.js

@@ -8,12 +8,14 @@ import { emitError } from '../utils/error/emit-error.js';
 import { createWriteRefCaches } from './db-ops/_shared/create-write-ref-caches.js';
 import { retrySetting } from './db-ops/_shared/retry-setting.js';
 import { replaceAnalysisViolations as replaceAnalysisViolationsOp } from './db-ops/analysis/replace-analysis-violations.js';
+import { replacePageTemplates as replacePageTemplatesOp } from './db-ops/analysis/replace-page-templates.js';
 import { getAnchorsOnPage as getAnchorsOnPageOp } from './db-ops/anchors/get-anchors-on-page.js';
 import { getBaseUrl as getBaseUrlOp } from './db-ops/config/get-base-url.js';
 import { getConfig as getConfigOp } from './db-ops/config/get-config.js';
 import { getName as getNameOp } from './db-ops/config/get-name.js';
 import { setConfig as setConfigOp } from './db-ops/config/set-config.js';
 import { updateConfig as updateConfigOp } from './db-ops/config/update-config.js';
+import { replaceConsoleLogs as replaceConsoleLogsOp } from './db-ops/console-logs/replace-console-logs.js';
 import { insertCrawlError as insertCrawlErrorOp } from './db-ops/errors/insert-crawl-error.js';
 import { insertPageError as insertPageErrorOp } from './db-ops/errors/insert-page-error.js';
 import { listDnsBurnedHostCandidates as listDnsBurnedHostCandidatesOp } from './db-ops/errors/list-dns-burned-host-candidates.js';
@@ -22,8 +24,19 @@ import { recordInventoryRun as recordInventoryRunOp } from './db-ops/inventory/r
 import { checkpoint as checkpointOp } from './db-ops/lifecycle/checkpoint.js';
 import { destroy as destroyOp } from './db-ops/lifecycle/destroy.js';
 import { init as initOp } from './db-ops/lifecycle/init.js';
+import { getAudiosOfPage as getAudiosOfPageOp } from './db-ops/meta/get-audios-of-page.js';
+import { getButtonsOfPage as getButtonsOfPageOp } from './db-ops/meta/get-buttons-of-page.js';
+import { getCanvasesOfPage as getCanvasesOfPageOp } from './db-ops/meta/get-canvases-of-page.js';
+import { getHeadingsOfPage as getHeadingsOfPageOp } from './db-ops/meta/get-headings-of-page.js';
+import { getIframesOfPage as getIframesOfPageOp } from './db-ops/meta/get-iframes-of-page.js';
 import { getJsonLdOfPage as getJsonLdOfPageOp } from './db-ops/meta/get-jsonld-of-page.js';
+import { getMainContentImagesOfPage as getMainContentImagesOfPageOp } from './db-ops/meta/get-main-content-images-of-page.js';
+import { getMainContentTablesOfPage as getMainContentTablesOfPageOp } from './db-ops/meta/get-main-content-tables-of-page.js';
 import { getTagsOfPage as getTagsOfPageOp } from './db-ops/meta/get-tags-of-page.js';
+import { getVideosOfPage as getVideosOfPageOp } from './db-ops/meta/get-videos-of-page.js';
+import { closeNetworkOutage as closeNetworkOutageOp } from './db-ops/outages/close-network-outage.js';
+import { insertNetworkOutage as insertNetworkOutageOp } from './db-ops/outages/insert-network-outage.js';
+import { listNetworkOutages as listNetworkOutagesOp } from './db-ops/outages/list-network-outages.js';
 import { setUrlOrder as setUrlOrderOp } from './db-ops/pages/order/set-url-order.js';
 import { getCrawlingState as getCrawlingStateOp } from './db-ops/pages/read/get-crawling-state.js';
 import { getExistingPageUrls as getExistingPageUrlsOp } from './db-ops/pages/read/get-existing-page-urls.js';
@@ -112,6 +125,15 @@ export class Database extends EventEmitter {
     async checkpoint() {
         await checkpointOp(this.#instance);
     }
+    /**
+     * Closes an outage row by stamping `ended_at` — a no-op if the row is
+     * already closed. Delegates to {@link closeNetworkOutageOp}.
+     * @param id - The `network_outages.id` to close.
+     * @param endedAt - Epoch ms the outage is considered to have ended.
+     */
+    async closeNetworkOutage(id, endedAt) {
+        return emitErrorAndRetry(this, 'Database.closeNetworkOutage', async () => await closeNetworkOutageOp(this.#instance, id, endedAt), retrySetting);
+    }
     /**
      * Destroys the database connection, releasing all pooled resources.
      * Delegates to {@link destroyOp}.
@@ -128,6 +150,14 @@ export class Database extends EventEmitter {
     async getAnchorsOnPage(pageId) {
         return emitErrorAndRetry(this, 'Database.getAnchorsOnPage', async () => await getAnchorsOnPageOp(this.#instance, pageId), retrySetting);
     }
+    /**
+     * Retrieves all `page_main_content_audios` rows for the given page id.
+     * Delegates to {@link getAudiosOfPageOp}.
+     * @param pageId
+     */
+    async getAudiosOfPage(pageId) {
+        return emitErrorAndRetry(this, 'Database.getAudiosOfPage', async () => await getAudiosOfPageOp(this.#instance, pageId), retrySetting);
+    }
     /**
      * Retrieves the base URL of the crawl session from the `info` table.
      * Delegates to {@link getBaseUrlOp}.
@@ -137,6 +167,22 @@ export class Database extends EventEmitter {
     async getBaseUrl() {
         return emitErrorAndRetry(this, 'Database.getBaseUrl', async () => await getBaseUrlOp(this.#instance), retrySetting);
     }
+    /**
+     * Retrieves all `page_main_content_buttons` rows for the given page id.
+     * Delegates to {@link getButtonsOfPageOp}.
+     * @param pageId
+     */
+    async getButtonsOfPage(pageId) {
+        return emitErrorAndRetry(this, 'Database.getButtonsOfPage', async () => await getButtonsOfPageOp(this.#instance, pageId), retrySetting);
+    }
+    /**
+     * Retrieves all `page_main_content_canvases` rows for the given page id.
+     * Delegates to {@link getCanvasesOfPageOp}.
+     * @param pageId
+     */
+    async getCanvasesOfPage(pageId) {
+        return emitErrorAndRetry(this, 'Database.getCanvasesOfPage', async () => await getCanvasesOfPageOp(this.#instance, pageId), retrySetting);
+    }
     /**
      * Retrieves the full crawl configuration from the `info` table.
      * Delegates to {@link getConfigOp}.
@@ -174,6 +220,14 @@ export class Database extends EventEmitter {
     async getExistingResourceUrls(urls) {
         return emitError(this, 'Database.getExistingResourceUrls', async () => await getExistingResourceUrlsOp(this.#instance, urls));
     }
+    /**
+     * Retrieves all `page_main_content_headings` rows for the given page id.
+     * Delegates to {@link getHeadingsOfPageOp}.
+     * @param pageId
+     */
+    async getHeadingsOfPage(pageId) {
+        return emitErrorAndRetry(this, 'Database.getHeadingsOfPage', async () => await getHeadingsOfPageOp(this.#instance, pageId), retrySetting);
+    }
     /**
      * Reads the HTML snapshot stored as a zstd-compressed BLOB for the given page.
      * Delegates to {@link getHtmlOfPageByIdOp}.
@@ -183,6 +237,14 @@ export class Database extends EventEmitter {
     async getHtmlOfPageById(pageId) {
         return emitErrorAndRetry(this, 'Database.getHtmlOfPageById', async () => await getHtmlOfPageByIdOp(this.#instance, pageId), retrySetting);
     }
+    /**
+     * Retrieves all `page_main_content_iframes` rows for the given page id.
+     * Delegates to {@link getIframesOfPageOp}.
+     * @param pageId
+     */
+    async getIframesOfPage(pageId) {
+        return emitErrorAndRetry(this, 'Database.getIframesOfPage', async () => await getIframesOfPageOp(this.#instance, pageId), retrySetting);
+    }
     /**
      * Retrieves all `page_jsonld` rows for the given page id, parsed back into
      * {@link JsonLdRow} shape. Delegates to {@link getJsonLdOfPageOp}.
@@ -200,6 +262,22 @@ export class Database extends EventEmitter {
     getKnex() {
         return this.#instance;
     }
+    /**
+     * Retrieves all `page_main_content_images` rows for the given page id.
+     * Delegates to {@link getMainContentImagesOfPageOp}.
+     * @param pageId
+     */
+    async getMainContentImagesOfPage(pageId) {
+        return emitErrorAndRetry(this, 'Database.getMainContentImagesOfPage', async () => await getMainContentImagesOfPageOp(this.#instance, pageId), retrySetting);
+    }
+    /**
+     * Retrieves all `page_main_content_tables` rows for the given page id.
+     * Delegates to {@link getMainContentTablesOfPageOp}.
+     * @param pageId
+     */
+    async getMainContentTablesOfPage(pageId) {
+        return emitErrorAndRetry(this, 'Database.getMainContentTablesOfPage', async () => await getMainContentTablesOfPageOp(this.#instance, pageId), retrySetting);
+    }
     /**
      * Retrieves the crawl session name from the `info` table.
      * Delegates to {@link getNameOp}.
@@ -326,6 +404,14 @@ export class Database extends EventEmitter {
     async getTagsOfPage(pageId) {
         return emitErrorAndRetry(this, 'Database.getTagsOfPage', async () => await getTagsOfPageOp(this.#instance, pageId), retrySetting);
     }
+    /**
+     * Retrieves all `page_main_content_videos` rows for the given page id.
+     * Delegates to {@link getVideosOfPageOp}.
+     * @param pageId
+     */
+    async getVideosOfPage(pageId) {
+        return emitErrorAndRetry(this, 'Database.getVideosOfPage', async () => await getVideosOfPageOp(this.#instance, pageId), retrySetting);
+    }
     /**
      * Records a crawler-level (`error` channel) failure into `crawl_errors`.
      * Delegates to {@link insertCrawlErrorOp}.
@@ -354,6 +440,15 @@ export class Database extends EventEmitter {
     async insertInventorySeeds(urls) {
         return emitErrorAndRetry(this, 'Database.insertInventorySeeds', async () => await insertInventorySeedsOp(this.#instance, this.#writeRefCaches, urls), retrySetting);
     }
+    /**
+     * Appends one open (`ended_at = NULL`) row to the `network_outages`
+     * journal. Delegates to {@link insertNetworkOutageOp}.
+     * @param params - The confirmed-outage fields to record.
+     * @returns The autoincremented `id` of the newly-inserted row.
+     */
+    async insertNetworkOutage(params) {
+        return emitErrorAndRetry(this, 'Database.insertNetworkOutage', async () => await insertNetworkOutageOp(this.#instance, params), retrySetting);
+    }
     /**
      * Records a partial scrape failure against the page identified by `url`.
      * Delegates to {@link insertPageErrorOp}.
@@ -392,6 +487,15 @@ export class Database extends EventEmitter {
     async listDnsBurnedHostCandidates() {
         return emitErrorAndRetry(this, 'Database.listDnsBurnedHostCandidates', async () => await listDnsBurnedHostCandidatesOp(this.#instance), retrySetting);
     }
+    /**
+     * Lists every recorded outage as a resolved {@link OutageWindow}.
+     * Delegates to {@link listNetworkOutagesOp}.
+     * @returns Resolved outage windows, or `[]` on an archive that predates
+     *   `network_outages` or has recorded no outages.
+     */
+    async listNetworkOutages() {
+        return emitErrorAndRetry(this, 'Database.listNetworkOutages', async () => await listNetworkOutagesOp(this.#instance), retrySetting);
+    }
     /**
      * Appends one row to the `inventory_runs` audit log.
      * Delegates to {@link recordInventoryRunOp}.
@@ -421,6 +525,30 @@ export class Database extends EventEmitter {
     async replaceAnalysisViolations(violations) {
         return emitErrorAndRetry(this, 'Database.replaceAnalysisViolations', async () => await replaceAnalysisViolationsOp(this.#instance, violations), retrySetting);
     }
+    /**
+     * Replaces one page's `page_console_logs` rows with a freshly captured
+     * set of console messages / page errors. Delegates to
+     * {@link replaceConsoleLogsOp}.
+     * @param pageUrl - The originally-requested URL, normalised (`withoutHashAndAuth` form).
+     * @param redirectPaths - The redirect chain hops captured during fetch, in order.
+     * @param entries - The console log entries to persist.
+     */
+    async replaceConsoleLogs(pageUrl, redirectPaths, entries) {
+        return emitErrorAndRetry(this, 'Database.replaceConsoleLogs', async () => await replaceConsoleLogsOp(this.#instance, this.#writeRefCaches, pageUrl, redirectPaths, entries), retrySetting);
+    }
+    /**
+     * Replaces the stored DOM-structure template classification with a
+     * freshly generated set. Delegates to {@link replacePageTemplatesOp}.
+     * @param templateKeysByUrl - Page URL → template key.
+     * @param clusterReasonsByTemplateKey - Template key → cluster-selection
+     *   evidence, if the caller captured it.
+     */
+    async replacePageTemplates(templateKeysByUrl, clusterReasonsByTemplateKey) {
+        return emitErrorAndRetry(this, 'Database.replacePageTemplates', async () => await replacePageTemplatesOp(this.#instance, {
+            templateKeysByUrl,
+            clusterReasonsByTemplateKey,
+        }), retrySetting);
+    }
     /**
      * Promote previously-external pages whose URL falls under any of the new
      * scope entries back to a "needs scraping" state.

package/lib/archive/db-ops/_shared/clear-write-ref-caches.js

@@ -30,5 +30,6 @@ export function clearWriteRefCaches(caches) {
     caches.contentTypeIds.clear();
     caches.jsonIds.clear();
     caches.blobIds.clear();
+    caches.consoleLogIds.clear();
     caches.headers = null;
 }

package/lib/archive/db-ops/_shared/compress-payload.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Compresses raw bytes for storage in a `codec`/`size_raw`/`size_stored`
+ * BLOB column, matching the read side of
+ * {@link import('./decode-json-ref.js').decodeJsonRef} and
+ * {@link import('../../decode-html-blob.js').decodeStoredBlob}.
+ *
+ * The one shared primitive behind every `codec: 'zstd'` writer in the
+ * archive (`upsert-json-ref.ts`, `write-page-html-blob.ts`,
+ * `replace-page-templates.ts`) — a codec change (e.g. a brotli fallback, a
+ * different zstd level) only has to happen here. Callers that also need a
+ * content hash (`upsert-json-ref.ts`, `write-page-html-blob.ts`) compute it
+ * themselves from the same `rawBytes` they pass in here, rather than this
+ * function computing one not every caller needs.
+ * @param rawBytes - The raw (uncompressed) payload bytes.
+ * @returns The compressed body alongside the codec and size bookkeeping the
+ *   BLOB column's schema expects.
+ * @example
+ * const rawBytes = Buffer.from(JSON.stringify(reason), 'utf8');
+ * const { body, codec, sizeRaw, sizeStored } = compressPayload(rawBytes);
+ */
+export declare function compressPayload(rawBytes: Buffer): {
+    readonly body: Buffer;
+    readonly codec: 'zstd';
+    readonly sizeRaw: number;
+    readonly sizeStored: number;
+};

package/lib/archive/db-ops/_shared/compress-payload.js

@@ -0,0 +1,30 @@
+import { zstdCompressSync } from 'node:zlib';
+/**
+ * Compresses raw bytes for storage in a `codec`/`size_raw`/`size_stored`
+ * BLOB column, matching the read side of
+ * {@link import('./decode-json-ref.js').decodeJsonRef} and
+ * {@link import('../../decode-html-blob.js').decodeStoredBlob}.
+ *
+ * The one shared primitive behind every `codec: 'zstd'` writer in the
+ * archive (`upsert-json-ref.ts`, `write-page-html-blob.ts`,
+ * `replace-page-templates.ts`) — a codec change (e.g. a brotli fallback, a
+ * different zstd level) only has to happen here. Callers that also need a
+ * content hash (`upsert-json-ref.ts`, `write-page-html-blob.ts`) compute it
+ * themselves from the same `rawBytes` they pass in here, rather than this
+ * function computing one not every caller needs.
+ * @param rawBytes - The raw (uncompressed) payload bytes.
+ * @returns The compressed body alongside the codec and size bookkeeping the
+ *   BLOB column's schema expects.
+ * @example
+ * const rawBytes = Buffer.from(JSON.stringify(reason), 'utf8');
+ * const { body, codec, sizeRaw, sizeStored } = compressPayload(rawBytes);
+ */
+export function compressPayload(rawBytes) {
+    const compressed = zstdCompressSync(rawBytes);
+    return {
+        body: compressed,
+        codec: 'zstd',
+        sizeRaw: rawBytes.byteLength,
+        sizeStored: compressed.byteLength,
+    };
+}

package/lib/archive/db-ops/_shared/create-write-ref-caches.js

@@ -21,6 +21,7 @@ export function createWriteRefCaches() {
         contentTypeIds: new Map(),
         jsonIds: new Map(),
         blobIds: new Map(),
+        consoleLogIds: new Map(),
         headers: null,
     };
 }

package/lib/archive/db-ops/_shared/types.d.ts

@@ -42,6 +42,8 @@ export interface WriteRefCaches {
     readonly jsonIds: Map<string, number>;
     /** `hex(content hash) → blob_refs.id`. */
     readonly blobIds: Map<string, number>;
+    /** `hex(content hash) → console_log_items.id`. */
+    readonly consoleLogIds: Map<string, number>;
     /**
      * Header dictionary caches (`header_name_refs` / `header_value_refs` /
      * `header_sets` id maps). `null` until the first header-set upsert

package/lib/archive/db-ops/_shared/upsert-json-ref.js

@@ -1,5 +1,5 @@
-import { zstdCompressSync } from 'node:zlib';
 import { computeContentHash } from '../../populate-ref-tables/compute-content-hash.js';
+import { compressPayload } from './compress-payload.js';
 /**
  * Resolves the `json_refs.id` for one JSON payload string, inserting the
  * deduplicated (hash-keyed) row when the payload is new.
@@ -27,11 +27,11 @@ export async function upsertJsonRef(qb, caches, jsonText) {
     if (cached !== undefined) {
         return cached;
     }
-    const compressed = zstdCompressSync(rawBytes);
+    const { body, sizeRaw, sizeStored } = compressPayload(rawBytes);
     const rows = await qb.raw(`INSERT INTO json_refs (hash, json_text, codec, size_raw, size_stored)
 		 VALUES (?, ?, 'zstd', ?, ?)
 		 ON CONFLICT(hash) DO UPDATE SET hash = hash
-		 RETURNING id`, [hash, compressed, rawBytes.byteLength, compressed.byteLength]);
+		 RETURNING id`, [hash, body, sizeRaw, sizeStored]);
     const first = rows[0];
     if (first === undefined) {
         throw new Error('upsertJsonRef: RETURNING yielded no row');

package/lib/archive/db-ops/analysis/replace-page-templates.d.ts

@@ -0,0 +1,29 @@
+import type { ReplacePageTemplatesParams } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Replaces the stored DOM-structure template classification (`--templates`)
+ * with a freshly generated set.
+ *
+ * Every classified page is a full-archive, all-or-nothing recomputation
+ * (see `@nitpicker/core`'s `classifyPageTemplates`), so this always deletes
+ * every existing row before inserting the new set — there is no per-page
+ * incremental update path, matching `replaceAnalysisViolations`'s
+ * whole-table replace shape. Unlike violations, a page whose URL can't be
+ * resolved back to a `content_items` row is silently skipped rather than
+ * treated as a hard failure: losing one page's template classification
+ * (e.g. a URL-normalization mismatch between the in-memory `Page.url.href`
+ * and the stored `url_refs.url`) should not discard the rest of a
+ * potentially multi-thousand-page classification run.
+ *
+ * `page_template_clusters` is always cleared alongside `page_templates`
+ * regardless of whether `clusterReasonsByTemplateKey` is passed — "no
+ * reason" must mean "not captured for this run", never "carry over the
+ * previous run's reason". Reason rows are inserted for every key in
+ * `clusterReasonsByTemplateKey` even if some have no surviving member page
+ * in `templateKeysByUrl` after URL-resolution skips above — harmless
+ * (nothing joins `page_template_clusters` back to `page_templates` by FK;
+ * see the table's own JSDoc), and simpler than cross-filtering the two maps.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param params - See {@link ReplacePageTemplatesParams}.
+ */
+export declare function replacePageTemplates(knex: Knex, params: ReplacePageTemplatesParams): Promise<void>;

package/lib/archive/db-ops/analysis/replace-page-templates.js

@@ -0,0 +1,84 @@
+import { eachSplitted } from '../../../utils/array/each-splitted.js';
+import { compressPayload } from '../_shared/compress-payload.js';
+/**
+ * Replaces the stored DOM-structure template classification (`--templates`)
+ * with a freshly generated set.
+ *
+ * Every classified page is a full-archive, all-or-nothing recomputation
+ * (see `@nitpicker/core`'s `classifyPageTemplates`), so this always deletes
+ * every existing row before inserting the new set — there is no per-page
+ * incremental update path, matching `replaceAnalysisViolations`'s
+ * whole-table replace shape. Unlike violations, a page whose URL can't be
+ * resolved back to a `content_items` row is silently skipped rather than
+ * treated as a hard failure: losing one page's template classification
+ * (e.g. a URL-normalization mismatch between the in-memory `Page.url.href`
+ * and the stored `url_refs.url`) should not discard the rest of a
+ * potentially multi-thousand-page classification run.
+ *
+ * `page_template_clusters` is always cleared alongside `page_templates`
+ * regardless of whether `clusterReasonsByTemplateKey` is passed — "no
+ * reason" must mean "not captured for this run", never "carry over the
+ * previous run's reason". Reason rows are inserted for every key in
+ * `clusterReasonsByTemplateKey` even if some have no surviving member page
+ * in `templateKeysByUrl` after URL-resolution skips above — harmless
+ * (nothing joins `page_template_clusters` back to `page_templates` by FK;
+ * see the table's own JSDoc), and simpler than cross-filtering the two maps.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param params - See {@link ReplacePageTemplatesParams}.
+ */
+export async function replacePageTemplates(knex, params) {
+    const { templateKeysByUrl, clusterReasonsByTemplateKey } = params;
+    // Compressing every reason is pure CPU work independent of the DB — done
+    // before opening the transaction below so it doesn't extend how long the
+    // SQLite write-lock is held for.
+    const reasonRows = clusterReasonsByTemplateKey && clusterReasonsByTemplateKey.size > 0
+        ? [...clusterReasonsByTemplateKey].map(([templateKey, reason]) => {
+            const { body, codec, sizeRaw, sizeStored } = compressPayload(Buffer.from(JSON.stringify(reason), 'utf8'));
+            return {
+                template_key: templateKey,
+                member_count: reason.memberCount,
+                reason_json: body,
+                codec,
+                size_raw: sizeRaw,
+                size_stored: sizeStored,
+            };
+        })
+        : [];
+    await knex.transaction(async (trx) => {
+        await trx('page_templates').delete();
+        await trx('page_template_clusters').delete();
+        if (reasonRows.length > 0) {
+            await eachSplitted(reasonRows, 100, async (chunk) => {
+                await trx('page_template_clusters').insert(chunk);
+            });
+        }
+        if (templateKeysByUrl.size === 0) {
+            return;
+        }
+        const urls = [...templateKeysByUrl.keys()];
+        const pageIdByUrl = new Map();
+        await eachSplitted(urls, 500, async (chunk) => {
+            const pageRows = await trx('content_items')
+                .join('url_refs', 'url_refs.id', 'content_items.url_id')
+                .select('content_items.id as id', 'url_refs.url as url')
+                .whereIn('url_refs.url', chunk);
+            for (const row of pageRows) {
+                pageIdByUrl.set(row.url, row.id);
+            }
+        });
+        const rows = [];
+        for (const [url, templateKey] of templateKeysByUrl) {
+            const pageId = pageIdByUrl.get(url);
+            if (pageId == null) {
+                continue;
+            }
+            rows.push({ page_id: pageId, template_key: templateKey });
+        }
+        if (rows.length === 0) {
+            return;
+        }
+        await eachSplitted(rows, 500, async (chunk) => {
+            await trx('page_templates').insert(chunk);
+        });
+    });
+}

package/lib/archive/db-ops/analysis/types.d.ts

@@ -0,0 +1,61 @@
+/**
+ * The six landmark types `@d-zero/page-cluster`'s `extractLandmarks` and
+ * `ClusterReason.landmarks` recognize.
+ */
+export type TemplateClusterLandmarkType = 'header' | 'footer' | 'nav' | 'aside' | 'form' | 'search';
+/**
+ * Why a Pass-0 block (one of possibly several that merged into a final
+ * cluster) was formed — mirrors `@d-zero/page-cluster`'s `BlockingReason`
+ * discriminated union.
+ */
+export type TemplateClusterBlockingReason = {
+    readonly kind: 'css';
+    readonly distinctiveStylesheetHrefs: readonly string[];
+} | {
+    readonly kind: 'path';
+    readonly pathKey: string;
+} | {
+    readonly kind: 'orphanMerge';
+    readonly pathKey: string;
+};
+/** One block's blocking key and the reason it was formed. */
+export interface TemplateClusterBlockingEvidence {
+    readonly blockKey: string;
+    readonly reason: TemplateClusterBlockingReason;
+}
+/** How common one landmark type is across a cluster's member pages. */
+export interface TemplateClusterLandmarkProfile {
+    readonly presenceRate: number;
+    readonly chromeRate: number;
+    readonly shellTokens: readonly string[];
+    readonly memberCountWithInstance: number;
+}
+/**
+ * nitpicker's own copy of `@d-zero/page-cluster`'s `ClusterReason` shape.
+ * Kept independent of the `@d-zero/page-cluster` package (rather than
+ * importing its type directly) so `@nitpicker/query` — which does not
+ * depend on `@d-zero/page-cluster` and is consumed by the browser-side
+ * viewer build — never has to add that dependency just to type a value
+ * read back out of the archive. `@d-zero/page-cluster`'s `ClusterReason`
+ * is structurally assignable to this type.
+ */
+export interface TemplateClusterReason {
+    readonly memberCount: number;
+    readonly blocking: readonly TemplateClusterBlockingEvidence[];
+    readonly structuralCoreTokens: readonly string[];
+    readonly landmarks: Partial<Record<TemplateClusterLandmarkType, TemplateClusterLandmarkProfile>>;
+    readonly siblingClusterKeys: readonly string[];
+}
+/**
+ * Params for {@link import('./replace-page-templates.js').replacePageTemplates}.
+ */
+export interface ReplacePageTemplatesParams {
+    /** Page URL → template key, as produced by `classifyPageTemplates`. */
+    readonly templateKeysByUrl: ReadonlyMap<string, string>;
+    /**
+     * Template key → `@d-zero/page-cluster`'s cluster-selection evidence for
+     * that key, as produced by `classifyPageTemplates`. Omitted entirely (not
+     * just empty) when the caller didn't request reasons.
+     */
+    readonly clusterReasonsByTemplateKey?: ReadonlyMap<string, TemplateClusterReason>;
+}

package/lib/archive/db-ops/analysis/types.js

@@ -0,0 +1 @@
+export {};

package/lib/archive/db-ops/config/info-column-allowlist.js

@@ -23,4 +23,5 @@ export const INFO_COLUMN_ALLOWLIST = new Set([
     'disableQueries',
     'userAgent',
     'ignoreRobots',
+    'mainContentSelector',
 ]);

package/lib/archive/db-ops/console-logs/compute-console-log-hash.d.ts

@@ -0,0 +1,19 @@
+import type { ConsoleLogContent } from './types.js';
+/**
+ * Computes the `console_log_items.hash` for one console log entry's content.
+ *
+ * Hashing over the raw content fields (not their resolved ref ids) keeps the
+ * dictionary row's identity independent of ref-table state, matching
+ * `text_refs` / `json_refs` / `url_refs`, whose hash columns are likewise
+ * computed from source content rather than downstream ids. The same
+ * message logged by a shared framework on every page therefore hashes to
+ * the same value regardless of insertion order or which page's scrape
+ * resolves the dictionary row first.
+ * @param entry - The content fields to hash. `argsJson` must already be
+ *   `stringifyConsoleLogArgs`'s output — this function does not
+ *   re-serialize `args` itself.
+ * @returns 32-byte SHA-256 hash, ready for the `hash BLOB UNIQUE` column.
+ * @example
+ * const hash = computeConsoleLogHash({ type: 'error', text: 'boom', argsJson: null });
+ */
+export declare function computeConsoleLogHash(entry: ConsoleLogContent): Buffer;

package/lib/archive/db-ops/console-logs/compute-console-log-hash.js

@@ -0,0 +1,30 @@
+import { computeContentHash } from '../../populate-ref-tables/compute-content-hash.js';
+/**
+ * Computes the `console_log_items.hash` for one console log entry's content.
+ *
+ * Hashing over the raw content fields (not their resolved ref ids) keeps the
+ * dictionary row's identity independent of ref-table state, matching
+ * `text_refs` / `json_refs` / `url_refs`, whose hash columns are likewise
+ * computed from source content rather than downstream ids. The same
+ * message logged by a shared framework on every page therefore hashes to
+ * the same value regardless of insertion order or which page's scrape
+ * resolves the dictionary row first.
+ * @param entry - The content fields to hash. `argsJson` must already be
+ *   `stringifyConsoleLogArgs`'s output — this function does not
+ *   re-serialize `args` itself.
+ * @returns 32-byte SHA-256 hash, ready for the `hash BLOB UNIQUE` column.
+ * @example
+ * const hash = computeConsoleLogHash({ type: 'error', text: 'boom', argsJson: null });
+ */
+export function computeConsoleLogHash(entry) {
+    const canonical = JSON.stringify([
+        entry.type,
+        entry.text,
+        entry.argsJson,
+        entry.location?.url ?? null,
+        entry.location?.lineNumber ?? null,
+        entry.location?.columnNumber ?? null,
+        entry.stack ?? null,
+    ]);
+    return computeContentHash(canonical);
+}

package/lib/archive/db-ops/console-logs/replace-console-logs.d.ts

@@ -0,0 +1,34 @@
+import type { WriteRefCaches } from '../_shared/types.js';
+import type { ConsoleLogEntry } from '@d-zero/beholder';
+import type { Knex } from 'knex';
+/**
+ * Replaces one page's `page_console_logs` rows wholesale with a freshly
+ * captured set of console messages / page errors (issue #228).
+ *
+ * Scoped-Replace, the same pattern as `replaceAnchorEdges` /
+ * `replaceImageItems`: the page's existing rows are deleted and the new
+ * set is inserted in the same transaction, so a re-scrape (`--append` /
+ * `--retry-failed` / re-render) never accumulates duplicate occurrences.
+ * Unlike those two, there is no non-empty guard on the DELETE here — the
+ * caller (`Crawler#handleConsoleLogs`) already skips emitting the event
+ * entirely when `entries` is empty, which is what keeps a degraded
+ * re-scrape from wiping out a prior good capture.
+ *
+ * `pageUrl` is the originally-requested URL (matching `updatePage`'s
+ * `page.url.withoutHashAndAuth` contract), not necessarily the page that
+ * ends up holding the content: `resolveRedirectChain` derives the same
+ * redirect destination `updatePage` writes content under, so console logs
+ * attach to the row that actually carries the page's `page_meta`, not to
+ * an empty placeholder for the pre-redirect URL.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param pageUrl - The originally-requested URL, normalised
+ *   (`withoutHashAndAuth` form).
+ * @param redirectPaths - The redirect chain hops captured during fetch, in
+ *   order (empty when the page was not redirected).
+ * @param entries - The console log entries to persist. Must be non-empty —
+ *   callers should skip calling this function entirely for an empty list.
+ * @example
+ * await replaceConsoleLogs(knex, caches, page.url.withoutHashAndAuth, page.redirectPaths, entries);
+ */
+export declare function replaceConsoleLogs(knex: Knex, caches: WriteRefCaches, pageUrl: string, redirectPaths: readonly string[], entries: readonly ConsoleLogEntry[]): Promise<void>;