@nitpicker/archive 0.0.0 → 0.25.1

package/lib/db-ops/config/get-base-url.js

@@ -0,0 +1,14 @@
+/**
+ * Retrieves the base URL of the crawl session from the `info` table.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The base URL string.
+ * @throws {Error} If no base URL is found in the database.
+ */
+export async function getBaseUrl(knex) {
+    const selected = await knex.select('baseUrl').from('info');
+    if (!selected[0]) {
+        throw new Error('No baseUrl');
+    }
+    const [{ baseUrl }] = selected;
+    return baseUrl || '';
+}

package/lib/db-ops/config/get-config.d.ts

@@ -0,0 +1,10 @@
+import type { Config } from '../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves the full crawl configuration from the `info` table.
+ * Deserializes JSON-encoded fields (`roots`, `excludes`, `excludeKeywords`, `excludeUrls`, `requestHeaderNames`).
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The parsed {@link Config} object.
+ * @throws {Error} If no configuration is found in the database.
+ */
+export declare function getConfig(knex: Knex): Promise<Config>;

package/lib/db-ops/config/get-config.js

@@ -0,0 +1,29 @@
+import { dbLog } from '../../debug.js';
+import { getJSON } from '../../get-json.js';
+/**
+ * Retrieves the full crawl configuration from the `info` table.
+ * Deserializes JSON-encoded fields (`roots`, `excludes`, `excludeKeywords`, `excludeUrls`, `requestHeaderNames`).
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The parsed {@link Config} object.
+ * @throws {Error} If no configuration is found in the database.
+ */
+export async function getConfig(knex) {
+    const [config] = await knex.select('*').from('info');
+    if (!config) {
+        throw new Error('No config');
+    }
+    const opt = {
+        ...config,
+        excludes: getJSON(config.excludes, []),
+        excludeKeywords: getJSON(config.excludeKeywords, []),
+        excludeUrls: getJSON(config.excludeUrls, []),
+        roots: getJSON(config.roots, []),
+        requestHeaderNames: getJSON(config.requestHeaderNames, []),
+        retry: config.retry ?? 3,
+        maxExcludedDepth: config.maxExcludedDepth ?? 0,
+    };
+    // @ts-expect-error — `id` is the primary key, not part of the public Config shape
+    delete opt.id;
+    dbLog('Table `info`: %O => %O', config, opt);
+    return opt;
+}

package/lib/db-ops/config/get-created-cwd.d.ts

@@ -0,0 +1,11 @@
+import type { Knex } from 'knex';
+/**
+ * Retrieves the working directory recorded when this stub's crawl session
+ * was created, so {@link import('../../archive.js').default.resume} can
+ * reconstruct the completed archive's output path independent of the
+ * directory `crawl --resume` happens to be invoked from.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The recorded cwd, or `null` for a stub created before this column
+ *   existed (or created by a code path that never stamps it, i.e. `resume`).
+ */
+export declare function getCreatedCwd(knex: Knex): Promise<string | null>;

package/lib/db-ops/config/get-created-cwd.js

@@ -0,0 +1,13 @@
+/**
+ * Retrieves the working directory recorded when this stub's crawl session
+ * was created, so {@link import('../../archive.js').default.resume} can
+ * reconstruct the completed archive's output path independent of the
+ * directory `crawl --resume` happens to be invoked from.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The recorded cwd, or `null` for a stub created before this column
+ *   existed (or created by a code path that never stamps it, i.e. `resume`).
+ */
+export async function getCreatedCwd(knex) {
+    const [row] = await knex.select('createdCwd').from('info');
+    return row?.createdCwd ?? null;
+}

package/lib/db-ops/config/get-name.d.ts

@@ -0,0 +1,8 @@
+import type { Knex } from 'knex';
+/**
+ * Retrieves the crawl session name from the `info` table.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The name string.
+ * @throws {Error} If no name is found in the database.
+ */
+export declare function getName(knex: Knex): Promise<string>;

package/lib/db-ops/config/get-name.js

@@ -0,0 +1,14 @@
+/**
+ * Retrieves the crawl session name from the `info` table.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The name string.
+ * @throws {Error} If no name is found in the database.
+ */
+export async function getName(knex) {
+    const selected = await knex.select('name').from('info');
+    if (!selected[0]) {
+        throw new Error('No name');
+    }
+    const [{ name }] = selected;
+    return name;
+}

package/lib/db-ops/config/info-column-allowlist.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Columns of the `info` table that `setConfig` / `updateConfig` are allowed to
+ * write. Any key outside this set is silently dropped so callers can splat a
+ * wider runtime config (with extras like `cwd`) without hitting "no such
+ * column" at the SQL layer.
+ */
+export declare const INFO_COLUMN_ALLOWLIST: ReadonlySet<string>;

package/lib/db-ops/config/info-column-allowlist.js

@@ -0,0 +1,29 @@
+/**
+ * Columns of the `info` table that `setConfig` / `updateConfig` are allowed to
+ * write. Any key outside this set is silently dropped so callers can splat a
+ * wider runtime config (with extras like `cwd`) without hitting "no such
+ * column" at the SQL layer.
+ */
+export const INFO_COLUMN_ALLOWLIST = new Set([
+    'version',
+    'name',
+    'baseUrl',
+    'roots',
+    'recursive',
+    'interval',
+    'image',
+    'fetchExternal',
+    'parallels',
+    'excludes',
+    'excludeKeywords',
+    'excludeUrls',
+    'maxExcludedDepth',
+    'retry',
+    'fromList',
+    'disableQueries',
+    'userAgent',
+    'ignoreRobots',
+    'mainContentSelector',
+    'requestHeaderNames',
+    'createdCwd',
+]);

package/lib/db-ops/config/info-json-columns.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Subset of {@link ./info-column-allowlist.ts} that is stored as a JSON-encoded
+ * string and therefore needs `JSON.stringify` on write.
+ */
+export declare const INFO_JSON_COLUMNS: ReadonlySet<string>;

package/lib/db-ops/config/info-json-columns.js

@@ -0,0 +1,11 @@
+/**
+ * Subset of {@link ./info-column-allowlist.ts} that is stored as a JSON-encoded
+ * string and therefore needs `JSON.stringify` on write.
+ */
+export const INFO_JSON_COLUMNS = new Set([
+    'roots',
+    'excludes',
+    'excludeKeywords',
+    'excludeUrls',
+    'requestHeaderNames',
+]);

package/lib/db-ops/config/set-config.d.ts

@@ -0,0 +1,12 @@
+import type { Config } from '../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Stores the crawl configuration in the `info` table.
+ * Only fields in {@link ./info-column-allowlist.ts} are forwarded — any extra
+ * runtime-only field on the input is silently dropped so callers can splat
+ * a wider config object without producing SQL errors. JSON-array fields
+ * are serialized via `JSON.stringify`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param config - The {@link Config} object to store.
+ */
+export declare function setConfig(knex: Knex, config: Config): Promise<number[]>;

package/lib/db-ops/config/set-config.js

@@ -0,0 +1,21 @@
+import { INFO_COLUMN_ALLOWLIST } from './info-column-allowlist.js';
+import { INFO_JSON_COLUMNS } from './info-json-columns.js';
+/**
+ * Stores the crawl configuration in the `info` table.
+ * Only fields in {@link ./info-column-allowlist.ts} are forwarded — any extra
+ * runtime-only field on the input is silently dropped so callers can splat
+ * a wider config object without producing SQL errors. JSON-array fields
+ * are serialized via `JSON.stringify`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param config - The {@link Config} object to store.
+ */
+export async function setConfig(knex, config) {
+    const payload = {};
+    for (const [key, value] of Object.entries(config)) {
+        if (!INFO_COLUMN_ALLOWLIST.has(key)) {
+            continue;
+        }
+        payload[key] = INFO_JSON_COLUMNS.has(key) ? JSON.stringify(value) : value;
+    }
+    return knex.from('info').insert(payload);
+}

package/lib/db-ops/config/update-config.d.ts

@@ -0,0 +1,17 @@
+import type { Config } from '../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Update the single row in the `info` table with a partial config patch.
+ *
+ * Used by the append flow to extend `roots` (and any other tweakable
+ * field) without replacing the entire row. JSON-array fields are serialized on
+ * the fly; primitive fields are written verbatim. Unspecified fields stay as-is.
+ *
+ * Unknown keys (anything outside the allow-list of `info`-table columns) are
+ * silently dropped instead of being passed to SQL, so callers that splat a
+ * wider runtime config (e.g. `CrawlConfig` with `cwd` / `executablePath`)
+ * cannot accidentally trigger a "no such column" SQL error.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param patch - Partial {@link Config} fields to overwrite. `undefined` values are skipped.
+ */
+export declare function updateConfig(knex: Knex, patch: Partial<Config>): Promise<void>;

package/lib/db-ops/config/update-config.js

@@ -0,0 +1,36 @@
+import { INFO_COLUMN_ALLOWLIST } from './info-column-allowlist.js';
+import { INFO_JSON_COLUMNS } from './info-json-columns.js';
+/**
+ * Update the single row in the `info` table with a partial config patch.
+ *
+ * Used by the append flow to extend `roots` (and any other tweakable
+ * field) without replacing the entire row. JSON-array fields are serialized on
+ * the fly; primitive fields are written verbatim. Unspecified fields stay as-is.
+ *
+ * Unknown keys (anything outside the allow-list of `info`-table columns) are
+ * silently dropped instead of being passed to SQL, so callers that splat a
+ * wider runtime config (e.g. `CrawlConfig` with `cwd` / `executablePath`)
+ * cannot accidentally trigger a "no such column" SQL error.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param patch - Partial {@link Config} fields to overwrite. `undefined` values are skipped.
+ */
+export async function updateConfig(knex, patch) {
+    const payload = {};
+    for (const [key, value] of Object.entries(patch)) {
+        if (value === undefined) {
+            continue;
+        }
+        if (!INFO_COLUMN_ALLOWLIST.has(key)) {
+            continue;
+        }
+        if (INFO_JSON_COLUMNS.has(key)) {
+            payload[key] = JSON.stringify(value);
+            continue;
+        }
+        payload[key] = value;
+    }
+    if (Object.keys(payload).length === 0) {
+        return;
+    }
+    await knex.from('info').update(payload);
+}

package/lib/db-ops/console-logs/compute-console-log-hash.d.ts

@@ -0,0 +1,19 @@
+import type { ConsoleLogContent } from './types.js';
+/**
+ * Computes the `console_log_items.hash` for one console log entry's content.
+ *
+ * Hashing over the raw content fields (not their resolved ref ids) keeps the
+ * dictionary row's identity independent of ref-table state, matching
+ * `text_refs` / `json_refs` / `url_refs`, whose hash columns are likewise
+ * computed from source content rather than downstream ids. The same
+ * message logged by a shared framework on every page therefore hashes to
+ * the same value regardless of insertion order or which page's scrape
+ * resolves the dictionary row first.
+ * @param entry - The content fields to hash. `argsJson` must already be
+ *   `stringifyConsoleLogArgs`'s output — this function does not
+ *   re-serialize `args` itself.
+ * @returns 32-byte SHA-256 hash, ready for the `hash BLOB UNIQUE` column.
+ * @example
+ * const hash = computeConsoleLogHash({ type: 'error', text: 'boom', argsJson: null });
+ */
+export declare function computeConsoleLogHash(entry: ConsoleLogContent): Buffer;

package/lib/db-ops/console-logs/compute-console-log-hash.js

@@ -0,0 +1,30 @@
+import { computeContentHash } from '../../populate-ref-tables/compute-content-hash.js';
+/**
+ * Computes the `console_log_items.hash` for one console log entry's content.
+ *
+ * Hashing over the raw content fields (not their resolved ref ids) keeps the
+ * dictionary row's identity independent of ref-table state, matching
+ * `text_refs` / `json_refs` / `url_refs`, whose hash columns are likewise
+ * computed from source content rather than downstream ids. The same
+ * message logged by a shared framework on every page therefore hashes to
+ * the same value regardless of insertion order or which page's scrape
+ * resolves the dictionary row first.
+ * @param entry - The content fields to hash. `argsJson` must already be
+ *   `stringifyConsoleLogArgs`'s output — this function does not
+ *   re-serialize `args` itself.
+ * @returns 32-byte SHA-256 hash, ready for the `hash BLOB UNIQUE` column.
+ * @example
+ * const hash = computeConsoleLogHash({ type: 'error', text: 'boom', argsJson: null });
+ */
+export function computeConsoleLogHash(entry) {
+    const canonical = JSON.stringify([
+        entry.type,
+        entry.text,
+        entry.argsJson,
+        entry.location?.url ?? null,
+        entry.location?.lineNumber ?? null,
+        entry.location?.columnNumber ?? null,
+        entry.stack ?? null,
+    ]);
+    return computeContentHash(canonical);
+}

package/lib/db-ops/console-logs/replace-console-logs.d.ts

@@ -0,0 +1,34 @@
+import type { ConsoleLogEntry } from '../../utils/types/types.js';
+import type { WriteRefCaches } from '../_shared/types.js';
+import type { Knex } from 'knex';
+/**
+ * Replaces one page's `page_console_logs` rows wholesale with a freshly
+ * captured set of console messages / page errors (issue #228).
+ *
+ * Scoped-Replace, the same pattern as `replaceAnchorEdges` /
+ * `replaceImageItems`: the page's existing rows are deleted and the new
+ * set is inserted in the same transaction, so a re-scrape (`--append` /
+ * `--retry-failed` / re-render) never accumulates duplicate occurrences.
+ * Unlike those two, there is no non-empty guard on the DELETE here — the
+ * caller (`Crawler#handleConsoleLogs`) already skips emitting the event
+ * entirely when `entries` is empty, which is what keeps a degraded
+ * re-scrape from wiping out a prior good capture.
+ *
+ * `pageUrl` is the originally-requested URL (matching `updatePage`'s
+ * `page.url.withoutHashAndAuth` contract), not necessarily the page that
+ * ends up holding the content: `resolveRedirectChain` derives the same
+ * redirect destination `updatePage` writes content under, so console logs
+ * attach to the row that actually carries the page's `page_meta`, not to
+ * an empty placeholder for the pre-redirect URL.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param pageUrl - The originally-requested URL, normalised
+ *   (`withoutHashAndAuth` form).
+ * @param redirectPaths - The redirect chain hops captured during fetch, in
+ *   order (empty when the page was not redirected).
+ * @param entries - The console log entries to persist. Must be non-empty —
+ *   callers should skip calling this function entirely for an empty list.
+ * @example
+ * await replaceConsoleLogs(knex, caches, page.url.withoutHashAndAuth, page.redirectPaths, entries);
+ */
+export declare function replaceConsoleLogs(knex: Knex, caches: WriteRefCaches, pageUrl: string, redirectPaths: readonly string[], entries: readonly ConsoleLogEntry[]): Promise<void>;

package/lib/db-ops/console-logs/replace-console-logs.js

@@ -0,0 +1,123 @@
+import { upsertTextRefs } from '../../populate-entity-tables/upsert-text-refs.js';
+import { resolveRedirectChain } from '../../resolve-redirect-chain.js';
+import { clearWriteRefCaches } from '../_shared/clear-write-ref-caches.js';
+import { resolveContentItemId } from '../_shared/resolve-content-item-id.js';
+import { upsertJsonRef } from '../_shared/upsert-json-ref.js';
+import { upsertUrlRef } from '../_shared/upsert-url-ref.js';
+import { computeConsoleLogHash } from './compute-console-log-hash.js';
+import { stringifyConsoleLogArgs } from './stringify-console-log-args.js';
+import { upsertConsoleLogItem } from './upsert-console-log-item.js';
+/** Rows sent per `INSERT INTO page_console_logs ... VALUES (...)` statement. */
+const INSERT_CHUNK_SIZE = 500;
+/**
+ * Replaces one page's `page_console_logs` rows wholesale with a freshly
+ * captured set of console messages / page errors (issue #228).
+ *
+ * Scoped-Replace, the same pattern as `replaceAnchorEdges` /
+ * `replaceImageItems`: the page's existing rows are deleted and the new
+ * set is inserted in the same transaction, so a re-scrape (`--append` /
+ * `--retry-failed` / re-render) never accumulates duplicate occurrences.
+ * Unlike those two, there is no non-empty guard on the DELETE here — the
+ * caller (`Crawler#handleConsoleLogs`) already skips emitting the event
+ * entirely when `entries` is empty, which is what keeps a degraded
+ * re-scrape from wiping out a prior good capture.
+ *
+ * `pageUrl` is the originally-requested URL (matching `updatePage`'s
+ * `page.url.withoutHashAndAuth` contract), not necessarily the page that
+ * ends up holding the content: `resolveRedirectChain` derives the same
+ * redirect destination `updatePage` writes content under, so console logs
+ * attach to the row that actually carries the page's `page_meta`, not to
+ * an empty placeholder for the pre-redirect URL.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param pageUrl - The originally-requested URL, normalised
+ *   (`withoutHashAndAuth` form).
+ * @param redirectPaths - The redirect chain hops captured during fetch, in
+ *   order (empty when the page was not redirected).
+ * @param entries - The console log entries to persist. Must be non-empty —
+ *   callers should skip calling this function entirely for an empty list.
+ * @example
+ * await replaceConsoleLogs(knex, caches, page.url.withoutHashAndAuth, page.redirectPaths, entries);
+ */
+export async function replaceConsoleLogs(knex, caches, pageUrl, redirectPaths, entries) {
+    const { destUrl } = resolveRedirectChain(pageUrl, redirectPaths);
+    try {
+        await knex.transaction(async (trx) => {
+            const pageId = await resolveContentItemId(trx, caches, destUrl);
+            await trx('page_console_logs').where('pageId', pageId).delete();
+            const texts = new Set();
+            for (const entry of entries) {
+                // `text_refs` never stores the empty string (its upsert
+                // treats `''` as "nothing to dedupe" and skips it) — a
+                // `console.log()` call with zero arguments reports `text:
+                // ''`, so that entry resolves to `textId: null` below
+                // instead of looking it up.
+                if (entry.text !== '') {
+                    texts.add(entry.text);
+                }
+                if (entry.stack) {
+                    texts.add(entry.stack);
+                }
+            }
+            const textIds = await upsertTextRefs(trx, texts);
+            const rows = [];
+            for (const entry of entries) {
+                let textId = null;
+                if (entry.text !== '') {
+                    const resolved = textIds.get(entry.text);
+                    if (resolved === undefined) {
+                        throw new Error(`replaceConsoleLogs: text_refs id not resolved for "${entry.text}"`);
+                    }
+                    textId = resolved;
+                }
+                const stackTextId = entry.stack ? (textIds.get(entry.stack) ?? null) : null;
+                const argsJson = stringifyConsoleLogArgs(entry.args);
+                const argsJsonId = argsJson === null ? null : await upsertJsonRef(trx, caches, argsJson);
+                const locUrlId = entry.location?.url === undefined
+                    ? null
+                    : await upsertUrlRef(trx, caches, entry.location.url);
+                const consoleLogId = await upsertConsoleLogItem(trx, caches, {
+                    hash: computeConsoleLogHash({
+                        type: entry.type,
+                        text: entry.text,
+                        argsJson,
+                        location: entry.location,
+                        stack: entry.stack,
+                    }),
+                    type: entry.type,
+                    textId,
+                    argsJsonId,
+                    locUrlId,
+                    locLine: entry.location?.lineNumber ?? null,
+                    locColumn: entry.location?.columnNumber ?? null,
+                    stackTextId,
+                });
+                rows.push({ pageId, consoleLogId, ts: entry.ts });
+            }
+            for (let index = 0; index < rows.length; index += INSERT_CHUNK_SIZE) {
+                await trx('page_console_logs').insert(rows.slice(index, index + INSERT_CHUNK_SIZE));
+            }
+            // Denormalised onto `page_meta` (the same write-once-at-scrape-time
+            // pattern as `tag_count` / `jsonld_count`) so the Pages list can
+            // display and sort by it without a live JOIN + COUNT per row.
+            // Computed from `entries` directly rather than re-querying the rows
+            // just inserted above — the data is already in hand. A `page_meta`
+            // row may not exist yet for a `'skipped'` / `'error'` scrape; the
+            // UPDATE then simply affects zero rows rather than throwing.
+            const errorCount = entries.filter((entry) => entry.type === 'error' || entry.type === 'pageerror').length;
+            await trx('page_meta').where('page_id', pageId).update({
+                console_error_count: errorCount,
+            });
+        });
+    }
+    catch (error) {
+        // A rolled-back transaction can leave ids cached that no longer
+        // correspond to any row (AUTOINCREMENT never rewinds) — same
+        // cache-poisoning hazard `updatePage` / `recordRedirect` guard
+        // against. A full clear (not a partial one) is required because
+        // this function shares `caches` with every other write path on
+        // the same connection.
+        clearWriteRefCaches(caches);
+        throw error;
+    }
+}

package/lib/db-ops/console-logs/stringify-console-log-args.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Serializes a console call's argument array to a JSON string, or `null`
+ * when there is nothing worth storing.
+ *
+ * Two cases collapse to `null` rather than an empty/degenerate string:
+ * an empty `args` array (most commonly a `"pageerror"` entry, which
+ * beholder always reports with `args: []`) and a `JSON.stringify` failure
+ * (e.g. a circular reference in a logged object) — both mean "no
+ * additional structured payload beyond `text`", not "a malformed one".
+ * @param args - The `ConsoleLogEntry.args` array to serialize.
+ * @returns The JSON string, or `null` when empty or unserializable.
+ * @example
+ * stringifyConsoleLogArgs(['a', 1]); // '["a",1]'
+ * stringifyConsoleLogArgs([]); // null
+ */
+export declare function stringifyConsoleLogArgs(args: readonly unknown[]): string | null;

package/lib/db-ops/console-logs/stringify-console-log-args.js

@@ -0,0 +1,26 @@
+/**
+ * Serializes a console call's argument array to a JSON string, or `null`
+ * when there is nothing worth storing.
+ *
+ * Two cases collapse to `null` rather than an empty/degenerate string:
+ * an empty `args` array (most commonly a `"pageerror"` entry, which
+ * beholder always reports with `args: []`) and a `JSON.stringify` failure
+ * (e.g. a circular reference in a logged object) — both mean "no
+ * additional structured payload beyond `text`", not "a malformed one".
+ * @param args - The `ConsoleLogEntry.args` array to serialize.
+ * @returns The JSON string, or `null` when empty or unserializable.
+ * @example
+ * stringifyConsoleLogArgs(['a', 1]); // '["a",1]'
+ * stringifyConsoleLogArgs([]); // null
+ */
+export function stringifyConsoleLogArgs(args) {
+    if (args.length === 0) {
+        return null;
+    }
+    try {
+        return JSON.stringify(args);
+    }
+    catch {
+        return null;
+    }
+}

package/lib/db-ops/console-logs/types.d.ts

@@ -0,0 +1,38 @@
+/**
+ * The content fields of a `ConsoleLogEntry` that determine its identity in
+ * `console_log_items` — everything except `pageUrl` / `ts`, which are
+ * per-occurrence rather than per-content (see `page_console_logs`).
+ */
+export interface ConsoleLogContent {
+    type: string;
+    text: string;
+    /**
+     * Pre-stringified `args` (via `stringifyConsoleLogArgs`), or `null` for
+     * an empty/unserializable args array. Callers compute this once and
+     * reuse it both for hashing here and for the `json_refs` storage
+     * decision, rather than calling `stringifyConsoleLogArgs` on the same
+     * `args` array twice.
+     */
+    argsJson: string | null;
+    location?: {
+        url?: string;
+        lineNumber?: number;
+        columnNumber?: number;
+    };
+    stack?: string;
+}
+/**
+ * The resolved ref ids and scalar fields that make up one
+ * `console_log_items` row, computed by the caller before the upsert runs.
+ */
+export interface ConsoleLogItemRow {
+    hash: Buffer;
+    type: string;
+    /** `null` when `text` is the empty string — `text_refs` never stores `''`. */
+    textId: number | null;
+    argsJsonId: number | null;
+    locUrlId: number | null;
+    locLine: number | null;
+    locColumn: number | null;
+    stackTextId: number | null;
+}

package/lib/db-ops/console-logs/types.js

@@ -0,0 +1 @@
+export {};

package/lib/db-ops/console-logs/upsert-console-log-item.d.ts

@@ -0,0 +1,29 @@
+import type { ConsoleLogItemRow } from './types.js';
+import type { WriteRefCaches } from '../_shared/types.js';
+import type { Knex } from 'knex';
+/**
+ * Resolves the `console_log_items.id` for one console log entry's content,
+ * inserting the dictionary row when the hash is not yet known.
+ *
+ * Same no-op-update-for-`RETURNING` idiom as `upsertUrlRef` / `upsertJsonRef`:
+ * `INSERT ... ON CONFLICT(hash) DO UPDATE SET hash = hash RETURNING id`
+ * yields the existing row's id on a hash collision without a separate
+ * SELECT round trip, and (unlike `INSERT OR IGNORE ... RETURNING`) still
+ * returns a row when the insert itself is the no-op.
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ * @param caches - The connection's write-side id caches; mutated in place.
+ * @param row - The content hash plus every resolved ref id / scalar column.
+ * @returns The `console_log_items.id` of the existing or newly inserted row.
+ * @example
+ * const id = await upsertConsoleLogItem(trx, caches, {
+ *   hash: computeConsoleLogHash(entry),
+ *   type: entry.type,
+ *   textId,
+ *   argsJsonId: null,
+ *   locUrlId: null,
+ *   locLine: null,
+ *   locColumn: null,
+ *   stackTextId: null,
+ * });
+ */
+export declare function upsertConsoleLogItem(qb: Knex | Knex.Transaction, caches: WriteRefCaches, row: ConsoleLogItemRow): Promise<number>;

package/lib/db-ops/console-logs/upsert-console-log-item.js

@@ -0,0 +1,52 @@
+/**
+ * Resolves the `console_log_items.id` for one console log entry's content,
+ * inserting the dictionary row when the hash is not yet known.
+ *
+ * Same no-op-update-for-`RETURNING` idiom as `upsertUrlRef` / `upsertJsonRef`:
+ * `INSERT ... ON CONFLICT(hash) DO UPDATE SET hash = hash RETURNING id`
+ * yields the existing row's id on a hash collision without a separate
+ * SELECT round trip, and (unlike `INSERT OR IGNORE ... RETURNING`) still
+ * returns a row when the insert itself is the no-op.
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ * @param caches - The connection's write-side id caches; mutated in place.
+ * @param row - The content hash plus every resolved ref id / scalar column.
+ * @returns The `console_log_items.id` of the existing or newly inserted row.
+ * @example
+ * const id = await upsertConsoleLogItem(trx, caches, {
+ *   hash: computeConsoleLogHash(entry),
+ *   type: entry.type,
+ *   textId,
+ *   argsJsonId: null,
+ *   locUrlId: null,
+ *   locLine: null,
+ *   locColumn: null,
+ *   stackTextId: null,
+ * });
+ */
+export async function upsertConsoleLogItem(qb, caches, row) {
+    const hex = row.hash.toString('hex');
+    const cached = caches.consoleLogIds.get(hex);
+    if (cached !== undefined) {
+        return cached;
+    }
+    const rows = await qb.raw(`INSERT INTO console_log_items
+			(hash, type, text_id, args_json_id, loc_url_id, loc_line, loc_column, stack_text_id)
+		 VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+		 ON CONFLICT(hash) DO UPDATE SET hash = hash
+		 RETURNING id`, [
+        row.hash,
+        row.type,
+        row.textId,
+        row.argsJsonId,
+        row.locUrlId,
+        row.locLine,
+        row.locColumn,
+        row.stackTextId,
+    ]);
+    const first = rows[0];
+    if (first === undefined) {
+        throw new Error('upsertConsoleLogItem: RETURNING yielded no row');
+    }
+    caches.consoleLogIds.set(hex, first.id);
+    return first.id;
+}