@nitpicker/crawler 0.14.0 → 0.15.0

package/lib/archive/database.js

@@ -15,6 +15,7 @@ import { getConfig as getConfigOp } from './db-ops/config/get-config.js';
 import { getName as getNameOp } from './db-ops/config/get-name.js';
 import { setConfig as setConfigOp } from './db-ops/config/set-config.js';
 import { updateConfig as updateConfigOp } from './db-ops/config/update-config.js';
+import { replaceConsoleLogs as replaceConsoleLogsOp } from './db-ops/console-logs/replace-console-logs.js';
 import { insertCrawlError as insertCrawlErrorOp } from './db-ops/errors/insert-crawl-error.js';
 import { insertPageError as insertPageErrorOp } from './db-ops/errors/insert-page-error.js';
 import { listDnsBurnedHostCandidates as listDnsBurnedHostCandidatesOp } from './db-ops/errors/list-dns-burned-host-candidates.js';
@@ -33,6 +34,9 @@ import { getMainContentImagesOfPage as getMainContentImagesOfPageOp } from './db
 import { getMainContentTablesOfPage as getMainContentTablesOfPageOp } from './db-ops/meta/get-main-content-tables-of-page.js';
 import { getTagsOfPage as getTagsOfPageOp } from './db-ops/meta/get-tags-of-page.js';
 import { getVideosOfPage as getVideosOfPageOp } from './db-ops/meta/get-videos-of-page.js';
+import { closeNetworkOutage as closeNetworkOutageOp } from './db-ops/outages/close-network-outage.js';
+import { insertNetworkOutage as insertNetworkOutageOp } from './db-ops/outages/insert-network-outage.js';
+import { listNetworkOutages as listNetworkOutagesOp } from './db-ops/outages/list-network-outages.js';
 import { setUrlOrder as setUrlOrderOp } from './db-ops/pages/order/set-url-order.js';
 import { getCrawlingState as getCrawlingStateOp } from './db-ops/pages/read/get-crawling-state.js';
 import { getExistingPageUrls as getExistingPageUrlsOp } from './db-ops/pages/read/get-existing-page-urls.js';
@@ -121,6 +125,15 @@ export class Database extends EventEmitter {
     async checkpoint() {
         await checkpointOp(this.#instance);
     }
+    /**
+     * Closes an outage row by stamping `ended_at` — a no-op if the row is
+     * already closed. Delegates to {@link closeNetworkOutageOp}.
+     * @param id - The `network_outages.id` to close.
+     * @param endedAt - Epoch ms the outage is considered to have ended.
+     */
+    async closeNetworkOutage(id, endedAt) {
+        return emitErrorAndRetry(this, 'Database.closeNetworkOutage', async () => await closeNetworkOutageOp(this.#instance, id, endedAt), retrySetting);
+    }
     /**
      * Destroys the database connection, releasing all pooled resources.
      * Delegates to {@link destroyOp}.
@@ -427,6 +440,15 @@ export class Database extends EventEmitter {
     async insertInventorySeeds(urls) {
         return emitErrorAndRetry(this, 'Database.insertInventorySeeds', async () => await insertInventorySeedsOp(this.#instance, this.#writeRefCaches, urls), retrySetting);
     }
+    /**
+     * Appends one open (`ended_at = NULL`) row to the `network_outages`
+     * journal. Delegates to {@link insertNetworkOutageOp}.
+     * @param params - The confirmed-outage fields to record.
+     * @returns The autoincremented `id` of the newly-inserted row.
+     */
+    async insertNetworkOutage(params) {
+        return emitErrorAndRetry(this, 'Database.insertNetworkOutage', async () => await insertNetworkOutageOp(this.#instance, params), retrySetting);
+    }
     /**
      * Records a partial scrape failure against the page identified by `url`.
      * Delegates to {@link insertPageErrorOp}.
@@ -465,6 +487,15 @@ export class Database extends EventEmitter {
     async listDnsBurnedHostCandidates() {
         return emitErrorAndRetry(this, 'Database.listDnsBurnedHostCandidates', async () => await listDnsBurnedHostCandidatesOp(this.#instance), retrySetting);
     }
+    /**
+     * Lists every recorded outage as a resolved {@link OutageWindow}.
+     * Delegates to {@link listNetworkOutagesOp}.
+     * @returns Resolved outage windows, or `[]` on an archive that predates
+     *   `network_outages` or has recorded no outages.
+     */
+    async listNetworkOutages() {
+        return emitErrorAndRetry(this, 'Database.listNetworkOutages', async () => await listNetworkOutagesOp(this.#instance), retrySetting);
+    }
     /**
      * Appends one row to the `inventory_runs` audit log.
      * Delegates to {@link recordInventoryRunOp}.
@@ -494,13 +525,29 @@ export class Database extends EventEmitter {
     async replaceAnalysisViolations(violations) {
         return emitErrorAndRetry(this, 'Database.replaceAnalysisViolations', async () => await replaceAnalysisViolationsOp(this.#instance, violations), retrySetting);
     }
+    /**
+     * Replaces one page's `page_console_logs` rows with a freshly captured
+     * set of console messages / page errors. Delegates to
+     * {@link replaceConsoleLogsOp}.
+     * @param pageUrl - The originally-requested URL, normalised (`withoutHashAndAuth` form).
+     * @param redirectPaths - The redirect chain hops captured during fetch, in order.
+     * @param entries - The console log entries to persist.
+     */
+    async replaceConsoleLogs(pageUrl, redirectPaths, entries) {
+        return emitErrorAndRetry(this, 'Database.replaceConsoleLogs', async () => await replaceConsoleLogsOp(this.#instance, this.#writeRefCaches, pageUrl, redirectPaths, entries), retrySetting);
+    }
     /**
      * Replaces the stored DOM-structure template classification with a
      * freshly generated set. Delegates to {@link replacePageTemplatesOp}.
      * @param templateKeysByUrl - Page URL → template key.
-     */
-    async replacePageTemplates(templateKeysByUrl) {
-        return emitErrorAndRetry(this, 'Database.replacePageTemplates', async () => await replacePageTemplatesOp(this.#instance, templateKeysByUrl), retrySetting);
+     * @param clusterReasonsByTemplateKey - Template key → cluster-selection
+     *   evidence, if the caller captured it.
+     */
+    async replacePageTemplates(templateKeysByUrl, clusterReasonsByTemplateKey) {
+        return emitErrorAndRetry(this, 'Database.replacePageTemplates', async () => await replacePageTemplatesOp(this.#instance, {
+            templateKeysByUrl,
+            clusterReasonsByTemplateKey,
+        }), retrySetting);
     }
     /**
      * Promote previously-external pages whose URL falls under any of the new

package/lib/archive/db-ops/_shared/clear-write-ref-caches.js

@@ -30,5 +30,6 @@ export function clearWriteRefCaches(caches) {
     caches.contentTypeIds.clear();
     caches.jsonIds.clear();
     caches.blobIds.clear();
+    caches.consoleLogIds.clear();
     caches.headers = null;
 }

package/lib/archive/db-ops/_shared/compress-payload.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Compresses raw bytes for storage in a `codec`/`size_raw`/`size_stored`
+ * BLOB column, matching the read side of
+ * {@link import('./decode-json-ref.js').decodeJsonRef} and
+ * {@link import('../../decode-html-blob.js').decodeStoredBlob}.
+ *
+ * The one shared primitive behind every `codec: 'zstd'` writer in the
+ * archive (`upsert-json-ref.ts`, `write-page-html-blob.ts`,
+ * `replace-page-templates.ts`) — a codec change (e.g. a brotli fallback, a
+ * different zstd level) only has to happen here. Callers that also need a
+ * content hash (`upsert-json-ref.ts`, `write-page-html-blob.ts`) compute it
+ * themselves from the same `rawBytes` they pass in here, rather than this
+ * function computing one not every caller needs.
+ * @param rawBytes - The raw (uncompressed) payload bytes.
+ * @returns The compressed body alongside the codec and size bookkeeping the
+ *   BLOB column's schema expects.
+ * @example
+ * const rawBytes = Buffer.from(JSON.stringify(reason), 'utf8');
+ * const { body, codec, sizeRaw, sizeStored } = compressPayload(rawBytes);
+ */
+export declare function compressPayload(rawBytes: Buffer): {
+    readonly body: Buffer;
+    readonly codec: 'zstd';
+    readonly sizeRaw: number;
+    readonly sizeStored: number;
+};

package/lib/archive/db-ops/_shared/compress-payload.js

@@ -0,0 +1,30 @@
+import { zstdCompressSync } from 'node:zlib';
+/**
+ * Compresses raw bytes for storage in a `codec`/`size_raw`/`size_stored`
+ * BLOB column, matching the read side of
+ * {@link import('./decode-json-ref.js').decodeJsonRef} and
+ * {@link import('../../decode-html-blob.js').decodeStoredBlob}.
+ *
+ * The one shared primitive behind every `codec: 'zstd'` writer in the
+ * archive (`upsert-json-ref.ts`, `write-page-html-blob.ts`,
+ * `replace-page-templates.ts`) — a codec change (e.g. a brotli fallback, a
+ * different zstd level) only has to happen here. Callers that also need a
+ * content hash (`upsert-json-ref.ts`, `write-page-html-blob.ts`) compute it
+ * themselves from the same `rawBytes` they pass in here, rather than this
+ * function computing one not every caller needs.
+ * @param rawBytes - The raw (uncompressed) payload bytes.
+ * @returns The compressed body alongside the codec and size bookkeeping the
+ *   BLOB column's schema expects.
+ * @example
+ * const rawBytes = Buffer.from(JSON.stringify(reason), 'utf8');
+ * const { body, codec, sizeRaw, sizeStored } = compressPayload(rawBytes);
+ */
+export function compressPayload(rawBytes) {
+    const compressed = zstdCompressSync(rawBytes);
+    return {
+        body: compressed,
+        codec: 'zstd',
+        sizeRaw: rawBytes.byteLength,
+        sizeStored: compressed.byteLength,
+    };
+}

package/lib/archive/db-ops/_shared/create-write-ref-caches.js

@@ -21,6 +21,7 @@ export function createWriteRefCaches() {
         contentTypeIds: new Map(),
         jsonIds: new Map(),
         blobIds: new Map(),
+        consoleLogIds: new Map(),
         headers: null,
     };
 }

package/lib/archive/db-ops/_shared/types.d.ts

@@ -42,6 +42,8 @@ export interface WriteRefCaches {
     readonly jsonIds: Map<string, number>;
     /** `hex(content hash) → blob_refs.id`. */
     readonly blobIds: Map<string, number>;
+    /** `hex(content hash) → console_log_items.id`. */
+    readonly consoleLogIds: Map<string, number>;
     /**
      * Header dictionary caches (`header_name_refs` / `header_value_refs` /
      * `header_sets` id maps). `null` until the first header-set upsert

package/lib/archive/db-ops/_shared/upsert-json-ref.js

@@ -1,5 +1,5 @@
-import { zstdCompressSync } from 'node:zlib';
 import { computeContentHash } from '../../populate-ref-tables/compute-content-hash.js';
+import { compressPayload } from './compress-payload.js';
 /**
  * Resolves the `json_refs.id` for one JSON payload string, inserting the
  * deduplicated (hash-keyed) row when the payload is new.
@@ -27,11 +27,11 @@ export async function upsertJsonRef(qb, caches, jsonText) {
     if (cached !== undefined) {
         return cached;
     }
-    const compressed = zstdCompressSync(rawBytes);
+    const { body, sizeRaw, sizeStored } = compressPayload(rawBytes);
     const rows = await qb.raw(`INSERT INTO json_refs (hash, json_text, codec, size_raw, size_stored)
 		 VALUES (?, ?, 'zstd', ?, ?)
 		 ON CONFLICT(hash) DO UPDATE SET hash = hash
-		 RETURNING id`, [hash, compressed, rawBytes.byteLength, compressed.byteLength]);
+		 RETURNING id`, [hash, body, sizeRaw, sizeStored]);
     const first = rows[0];
     if (first === undefined) {
         throw new Error('upsertJsonRef: RETURNING yielded no row');

package/lib/archive/db-ops/analysis/replace-page-templates.d.ts

@@ -1,3 +1,4 @@
+import type { ReplacePageTemplatesParams } from './types.js';
 import type { Knex } from 'knex';
 /**
  * Replaces the stored DOM-structure template classification (`--templates`)
@@ -13,8 +14,16 @@ import type { Knex } from 'knex';
  * (e.g. a URL-normalization mismatch between the in-memory `Page.url.href`
  * and the stored `url_refs.url`) should not discard the rest of a
  * potentially multi-thousand-page classification run.
+ *
+ * `page_template_clusters` is always cleared alongside `page_templates`
+ * regardless of whether `clusterReasonsByTemplateKey` is passed — "no
+ * reason" must mean "not captured for this run", never "carry over the
+ * previous run's reason". Reason rows are inserted for every key in
+ * `clusterReasonsByTemplateKey` even if some have no surviving member page
+ * in `templateKeysByUrl` after URL-resolution skips above — harmless
+ * (nothing joins `page_template_clusters` back to `page_templates` by FK;
+ * see the table's own JSDoc), and simpler than cross-filtering the two maps.
  * @param knex - Knex query builder connected to the archive DB.
- * @param templateKeysByUrl - Page URL → template key, as produced by
- *   `classifyPageTemplates`.
+ * @param params - See {@link ReplacePageTemplatesParams}.
  */
-export declare function replacePageTemplates(knex: Knex, templateKeysByUrl: ReadonlyMap<string, string>): Promise<void>;
+export declare function replacePageTemplates(knex: Knex, params: ReplacePageTemplatesParams): Promise<void>;

package/lib/archive/db-ops/analysis/replace-page-templates.js

@@ -1,4 +1,5 @@
 import { eachSplitted } from '../../../utils/array/each-splitted.js';
+import { compressPayload } from '../_shared/compress-payload.js';
 /**
  * Replaces the stored DOM-structure template classification (`--templates`)
  * with a freshly generated set.
@@ -13,13 +14,44 @@ import { eachSplitted } from '../../../utils/array/each-splitted.js';
  * (e.g. a URL-normalization mismatch between the in-memory `Page.url.href`
  * and the stored `url_refs.url`) should not discard the rest of a
  * potentially multi-thousand-page classification run.
+ *
+ * `page_template_clusters` is always cleared alongside `page_templates`
+ * regardless of whether `clusterReasonsByTemplateKey` is passed — "no
+ * reason" must mean "not captured for this run", never "carry over the
+ * previous run's reason". Reason rows are inserted for every key in
+ * `clusterReasonsByTemplateKey` even if some have no surviving member page
+ * in `templateKeysByUrl` after URL-resolution skips above — harmless
+ * (nothing joins `page_template_clusters` back to `page_templates` by FK;
+ * see the table's own JSDoc), and simpler than cross-filtering the two maps.
  * @param knex - Knex query builder connected to the archive DB.
- * @param templateKeysByUrl - Page URL → template key, as produced by
- *   `classifyPageTemplates`.
+ * @param params - See {@link ReplacePageTemplatesParams}.
  */
-export async function replacePageTemplates(knex, templateKeysByUrl) {
+export async function replacePageTemplates(knex, params) {
+    const { templateKeysByUrl, clusterReasonsByTemplateKey } = params;
+    // Compressing every reason is pure CPU work independent of the DB — done
+    // before opening the transaction below so it doesn't extend how long the
+    // SQLite write-lock is held for.
+    const reasonRows = clusterReasonsByTemplateKey && clusterReasonsByTemplateKey.size > 0
+        ? [...clusterReasonsByTemplateKey].map(([templateKey, reason]) => {
+            const { body, codec, sizeRaw, sizeStored } = compressPayload(Buffer.from(JSON.stringify(reason), 'utf8'));
+            return {
+                template_key: templateKey,
+                member_count: reason.memberCount,
+                reason_json: body,
+                codec,
+                size_raw: sizeRaw,
+                size_stored: sizeStored,
+            };
+        })
+        : [];
     await knex.transaction(async (trx) => {
         await trx('page_templates').delete();
+        await trx('page_template_clusters').delete();
+        if (reasonRows.length > 0) {
+            await eachSplitted(reasonRows, 100, async (chunk) => {
+                await trx('page_template_clusters').insert(chunk);
+            });
+        }
         if (templateKeysByUrl.size === 0) {
             return;
         }

package/lib/archive/db-ops/analysis/types.d.ts

@@ -0,0 +1,61 @@
+/**
+ * The six landmark types `@d-zero/page-cluster`'s `extractLandmarks` and
+ * `ClusterReason.landmarks` recognize.
+ */
+export type TemplateClusterLandmarkType = 'header' | 'footer' | 'nav' | 'aside' | 'form' | 'search';
+/**
+ * Why a Pass-0 block (one of possibly several that merged into a final
+ * cluster) was formed — mirrors `@d-zero/page-cluster`'s `BlockingReason`
+ * discriminated union.
+ */
+export type TemplateClusterBlockingReason = {
+    readonly kind: 'css';
+    readonly distinctiveStylesheetHrefs: readonly string[];
+} | {
+    readonly kind: 'path';
+    readonly pathKey: string;
+} | {
+    readonly kind: 'orphanMerge';
+    readonly pathKey: string;
+};
+/** One block's blocking key and the reason it was formed. */
+export interface TemplateClusterBlockingEvidence {
+    readonly blockKey: string;
+    readonly reason: TemplateClusterBlockingReason;
+}
+/** How common one landmark type is across a cluster's member pages. */
+export interface TemplateClusterLandmarkProfile {
+    readonly presenceRate: number;
+    readonly chromeRate: number;
+    readonly shellTokens: readonly string[];
+    readonly memberCountWithInstance: number;
+}
+/**
+ * nitpicker's own copy of `@d-zero/page-cluster`'s `ClusterReason` shape.
+ * Kept independent of the `@d-zero/page-cluster` package (rather than
+ * importing its type directly) so `@nitpicker/query` — which does not
+ * depend on `@d-zero/page-cluster` and is consumed by the browser-side
+ * viewer build — never has to add that dependency just to type a value
+ * read back out of the archive. `@d-zero/page-cluster`'s `ClusterReason`
+ * is structurally assignable to this type.
+ */
+export interface TemplateClusterReason {
+    readonly memberCount: number;
+    readonly blocking: readonly TemplateClusterBlockingEvidence[];
+    readonly structuralCoreTokens: readonly string[];
+    readonly landmarks: Partial<Record<TemplateClusterLandmarkType, TemplateClusterLandmarkProfile>>;
+    readonly siblingClusterKeys: readonly string[];
+}
+/**
+ * Params for {@link import('./replace-page-templates.js').replacePageTemplates}.
+ */
+export interface ReplacePageTemplatesParams {
+    /** Page URL → template key, as produced by `classifyPageTemplates`. */
+    readonly templateKeysByUrl: ReadonlyMap<string, string>;
+    /**
+     * Template key → `@d-zero/page-cluster`'s cluster-selection evidence for
+     * that key, as produced by `classifyPageTemplates`. Omitted entirely (not
+     * just empty) when the caller didn't request reasons.
+     */
+    readonly clusterReasonsByTemplateKey?: ReadonlyMap<string, TemplateClusterReason>;
+}

package/lib/archive/db-ops/analysis/types.js

@@ -0,0 +1 @@
+export {};

package/lib/archive/db-ops/console-logs/compute-console-log-hash.d.ts

@@ -0,0 +1,19 @@
+import type { ConsoleLogContent } from './types.js';
+/**
+ * Computes the `console_log_items.hash` for one console log entry's content.
+ *
+ * Hashing over the raw content fields (not their resolved ref ids) keeps the
+ * dictionary row's identity independent of ref-table state, matching
+ * `text_refs` / `json_refs` / `url_refs`, whose hash columns are likewise
+ * computed from source content rather than downstream ids. The same
+ * message logged by a shared framework on every page therefore hashes to
+ * the same value regardless of insertion order or which page's scrape
+ * resolves the dictionary row first.
+ * @param entry - The content fields to hash. `argsJson` must already be
+ *   `stringifyConsoleLogArgs`'s output — this function does not
+ *   re-serialize `args` itself.
+ * @returns 32-byte SHA-256 hash, ready for the `hash BLOB UNIQUE` column.
+ * @example
+ * const hash = computeConsoleLogHash({ type: 'error', text: 'boom', argsJson: null });
+ */
+export declare function computeConsoleLogHash(entry: ConsoleLogContent): Buffer;

package/lib/archive/db-ops/console-logs/compute-console-log-hash.js

@@ -0,0 +1,30 @@
+import { computeContentHash } from '../../populate-ref-tables/compute-content-hash.js';
+/**
+ * Computes the `console_log_items.hash` for one console log entry's content.
+ *
+ * Hashing over the raw content fields (not their resolved ref ids) keeps the
+ * dictionary row's identity independent of ref-table state, matching
+ * `text_refs` / `json_refs` / `url_refs`, whose hash columns are likewise
+ * computed from source content rather than downstream ids. The same
+ * message logged by a shared framework on every page therefore hashes to
+ * the same value regardless of insertion order or which page's scrape
+ * resolves the dictionary row first.
+ * @param entry - The content fields to hash. `argsJson` must already be
+ *   `stringifyConsoleLogArgs`'s output — this function does not
+ *   re-serialize `args` itself.
+ * @returns 32-byte SHA-256 hash, ready for the `hash BLOB UNIQUE` column.
+ * @example
+ * const hash = computeConsoleLogHash({ type: 'error', text: 'boom', argsJson: null });
+ */
+export function computeConsoleLogHash(entry) {
+    const canonical = JSON.stringify([
+        entry.type,
+        entry.text,
+        entry.argsJson,
+        entry.location?.url ?? null,
+        entry.location?.lineNumber ?? null,
+        entry.location?.columnNumber ?? null,
+        entry.stack ?? null,
+    ]);
+    return computeContentHash(canonical);
+}

package/lib/archive/db-ops/console-logs/replace-console-logs.d.ts

@@ -0,0 +1,34 @@
+import type { WriteRefCaches } from '../_shared/types.js';
+import type { ConsoleLogEntry } from '@d-zero/beholder';
+import type { Knex } from 'knex';
+/**
+ * Replaces one page's `page_console_logs` rows wholesale with a freshly
+ * captured set of console messages / page errors (issue #228).
+ *
+ * Scoped-Replace, the same pattern as `replaceAnchorEdges` /
+ * `replaceImageItems`: the page's existing rows are deleted and the new
+ * set is inserted in the same transaction, so a re-scrape (`--append` /
+ * `--retry-failed` / re-render) never accumulates duplicate occurrences.
+ * Unlike those two, there is no non-empty guard on the DELETE here — the
+ * caller (`Crawler#handleConsoleLogs`) already skips emitting the event
+ * entirely when `entries` is empty, which is what keeps a degraded
+ * re-scrape from wiping out a prior good capture.
+ *
+ * `pageUrl` is the originally-requested URL (matching `updatePage`'s
+ * `page.url.withoutHashAndAuth` contract), not necessarily the page that
+ * ends up holding the content: `resolveRedirectChain` derives the same
+ * redirect destination `updatePage` writes content under, so console logs
+ * attach to the row that actually carries the page's `page_meta`, not to
+ * an empty placeholder for the pre-redirect URL.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param pageUrl - The originally-requested URL, normalised
+ *   (`withoutHashAndAuth` form).
+ * @param redirectPaths - The redirect chain hops captured during fetch, in
+ *   order (empty when the page was not redirected).
+ * @param entries - The console log entries to persist. Must be non-empty —
+ *   callers should skip calling this function entirely for an empty list.
+ * @example
+ * await replaceConsoleLogs(knex, caches, page.url.withoutHashAndAuth, page.redirectPaths, entries);
+ */
+export declare function replaceConsoleLogs(knex: Knex, caches: WriteRefCaches, pageUrl: string, redirectPaths: readonly string[], entries: readonly ConsoleLogEntry[]): Promise<void>;

package/lib/archive/db-ops/console-logs/replace-console-logs.js

@@ -0,0 +1,123 @@
+import { upsertTextRefs } from '../../populate-entity-tables/upsert-text-refs.js';
+import { resolveRedirectChain } from '../../resolve-redirect-chain.js';
+import { clearWriteRefCaches } from '../_shared/clear-write-ref-caches.js';
+import { resolveContentItemId } from '../_shared/resolve-content-item-id.js';
+import { upsertJsonRef } from '../_shared/upsert-json-ref.js';
+import { upsertUrlRef } from '../_shared/upsert-url-ref.js';
+import { computeConsoleLogHash } from './compute-console-log-hash.js';
+import { stringifyConsoleLogArgs } from './stringify-console-log-args.js';
+import { upsertConsoleLogItem } from './upsert-console-log-item.js';
+/** Rows sent per `INSERT INTO page_console_logs ... VALUES (...)` statement. */
+const INSERT_CHUNK_SIZE = 500;
+/**
+ * Replaces one page's `page_console_logs` rows wholesale with a freshly
+ * captured set of console messages / page errors (issue #228).
+ *
+ * Scoped-Replace, the same pattern as `replaceAnchorEdges` /
+ * `replaceImageItems`: the page's existing rows are deleted and the new
+ * set is inserted in the same transaction, so a re-scrape (`--append` /
+ * `--retry-failed` / re-render) never accumulates duplicate occurrences.
+ * Unlike those two, there is no non-empty guard on the DELETE here — the
+ * caller (`Crawler#handleConsoleLogs`) already skips emitting the event
+ * entirely when `entries` is empty, which is what keeps a degraded
+ * re-scrape from wiping out a prior good capture.
+ *
+ * `pageUrl` is the originally-requested URL (matching `updatePage`'s
+ * `page.url.withoutHashAndAuth` contract), not necessarily the page that
+ * ends up holding the content: `resolveRedirectChain` derives the same
+ * redirect destination `updatePage` writes content under, so console logs
+ * attach to the row that actually carries the page's `page_meta`, not to
+ * an empty placeholder for the pre-redirect URL.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param pageUrl - The originally-requested URL, normalised
+ *   (`withoutHashAndAuth` form).
+ * @param redirectPaths - The redirect chain hops captured during fetch, in
+ *   order (empty when the page was not redirected).
+ * @param entries - The console log entries to persist. Must be non-empty —
+ *   callers should skip calling this function entirely for an empty list.
+ * @example
+ * await replaceConsoleLogs(knex, caches, page.url.withoutHashAndAuth, page.redirectPaths, entries);
+ */
+export async function replaceConsoleLogs(knex, caches, pageUrl, redirectPaths, entries) {
+    const { destUrl } = resolveRedirectChain(pageUrl, redirectPaths);
+    try {
+        await knex.transaction(async (trx) => {
+            const pageId = await resolveContentItemId(trx, caches, destUrl);
+            await trx('page_console_logs').where('pageId', pageId).delete();
+            const texts = new Set();
+            for (const entry of entries) {
+                // `text_refs` never stores the empty string (its upsert
+                // treats `''` as "nothing to dedupe" and skips it) — a
+                // `console.log()` call with zero arguments reports `text:
+                // ''`, so that entry resolves to `textId: null` below
+                // instead of looking it up.
+                if (entry.text !== '') {
+                    texts.add(entry.text);
+                }
+                if (entry.stack) {
+                    texts.add(entry.stack);
+                }
+            }
+            const textIds = await upsertTextRefs(trx, texts);
+            const rows = [];
+            for (const entry of entries) {
+                let textId = null;
+                if (entry.text !== '') {
+                    const resolved = textIds.get(entry.text);
+                    if (resolved === undefined) {
+                        throw new Error(`replaceConsoleLogs: text_refs id not resolved for "${entry.text}"`);
+                    }
+                    textId = resolved;
+                }
+                const stackTextId = entry.stack ? (textIds.get(entry.stack) ?? null) : null;
+                const argsJson = stringifyConsoleLogArgs(entry.args);
+                const argsJsonId = argsJson === null ? null : await upsertJsonRef(trx, caches, argsJson);
+                const locUrlId = entry.location?.url === undefined
+                    ? null
+                    : await upsertUrlRef(trx, caches, entry.location.url);
+                const consoleLogId = await upsertConsoleLogItem(trx, caches, {
+                    hash: computeConsoleLogHash({
+                        type: entry.type,
+                        text: entry.text,
+                        argsJson,
+                        location: entry.location,
+                        stack: entry.stack,
+                    }),
+                    type: entry.type,
+                    textId,
+                    argsJsonId,
+                    locUrlId,
+                    locLine: entry.location?.lineNumber ?? null,
+                    locColumn: entry.location?.columnNumber ?? null,
+                    stackTextId,
+                });
+                rows.push({ pageId, consoleLogId, ts: entry.ts });
+            }
+            for (let index = 0; index < rows.length; index += INSERT_CHUNK_SIZE) {
+                await trx('page_console_logs').insert(rows.slice(index, index + INSERT_CHUNK_SIZE));
+            }
+            // Denormalised onto `page_meta` (the same write-once-at-scrape-time
+            // pattern as `tag_count` / `jsonld_count`) so the Pages list can
+            // display and sort by it without a live JOIN + COUNT per row.
+            // Computed from `entries` directly rather than re-querying the rows
+            // just inserted above — the data is already in hand. A `page_meta`
+            // row may not exist yet for a `'skipped'` / `'error'` scrape; the
+            // UPDATE then simply affects zero rows rather than throwing.
+            const errorCount = entries.filter((entry) => entry.type === 'error' || entry.type === 'pageerror').length;
+            await trx('page_meta').where('page_id', pageId).update({
+                console_error_count: errorCount,
+            });
+        });
+    }
+    catch (error) {
+        // A rolled-back transaction can leave ids cached that no longer
+        // correspond to any row (AUTOINCREMENT never rewinds) — same
+        // cache-poisoning hazard `updatePage` / `recordRedirect` guard
+        // against. A full clear (not a partial one) is required because
+        // this function shares `caches` with every other write path on
+        // the same connection.
+        clearWriteRefCaches(caches);
+        throw error;
+    }
+}

package/lib/archive/db-ops/console-logs/stringify-console-log-args.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Serializes a console call's argument array to a JSON string, or `null`
+ * when there is nothing worth storing.
+ *
+ * Two cases collapse to `null` rather than an empty/degenerate string:
+ * an empty `args` array (most commonly a `"pageerror"` entry, which
+ * beholder always reports with `args: []`) and a `JSON.stringify` failure
+ * (e.g. a circular reference in a logged object) — both mean "no
+ * additional structured payload beyond `text`", not "a malformed one".
+ * @param args - The `ConsoleLogEntry.args` array to serialize.
+ * @returns The JSON string, or `null` when empty or unserializable.
+ * @example
+ * stringifyConsoleLogArgs(['a', 1]); // '["a",1]'
+ * stringifyConsoleLogArgs([]); // null
+ */
+export declare function stringifyConsoleLogArgs(args: readonly unknown[]): string | null;

package/lib/archive/db-ops/console-logs/stringify-console-log-args.js

@@ -0,0 +1,26 @@
+/**
+ * Serializes a console call's argument array to a JSON string, or `null`
+ * when there is nothing worth storing.
+ *
+ * Two cases collapse to `null` rather than an empty/degenerate string:
+ * an empty `args` array (most commonly a `"pageerror"` entry, which
+ * beholder always reports with `args: []`) and a `JSON.stringify` failure
+ * (e.g. a circular reference in a logged object) — both mean "no
+ * additional structured payload beyond `text`", not "a malformed one".
+ * @param args - The `ConsoleLogEntry.args` array to serialize.
+ * @returns The JSON string, or `null` when empty or unserializable.
+ * @example
+ * stringifyConsoleLogArgs(['a', 1]); // '["a",1]'
+ * stringifyConsoleLogArgs([]); // null
+ */
+export function stringifyConsoleLogArgs(args) {
+    if (args.length === 0) {
+        return null;
+    }
+    try {
+        return JSON.stringify(args);
+    }
+    catch {
+        return null;
+    }
+}

package/lib/archive/db-ops/console-logs/types.d.ts

@@ -0,0 +1,38 @@
+/**
+ * The content fields of a `ConsoleLogEntry` that determine its identity in
+ * `console_log_items` — everything except `pageUrl` / `ts`, which are
+ * per-occurrence rather than per-content (see `page_console_logs`).
+ */
+export interface ConsoleLogContent {
+    type: string;
+    text: string;
+    /**
+     * Pre-stringified `args` (via `stringifyConsoleLogArgs`), or `null` for
+     * an empty/unserializable args array. Callers compute this once and
+     * reuse it both for hashing here and for the `json_refs` storage
+     * decision, rather than calling `stringifyConsoleLogArgs` on the same
+     * `args` array twice.
+     */
+    argsJson: string | null;
+    location?: {
+        url?: string;
+        lineNumber?: number;
+        columnNumber?: number;
+    };
+    stack?: string;
+}
+/**
+ * The resolved ref ids and scalar fields that make up one
+ * `console_log_items` row, computed by the caller before the upsert runs.
+ */
+export interface ConsoleLogItemRow {
+    hash: Buffer;
+    type: string;
+    /** `null` when `text` is the empty string — `text_refs` never stores `''`. */
+    textId: number | null;
+    argsJsonId: number | null;
+    locUrlId: number | null;
+    locLine: number | null;
+    locColumn: number | null;
+    stackTextId: number | null;
+}

package/lib/archive/db-ops/console-logs/types.js

@@ -0,0 +1 @@
+export {};