npm - @nitpicker/crawler - Versions diffs - 0.14.0 → 0.16.0 - Mend

@nitpicker/crawler 0.14.0 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (155) hide show

package/lib/archive/body-hash/normalize-url-like-strings.js ADDED Viewed

@@ -0,0 +1,19 @@
+const INDEX_SUFFIX_PATTERN = /\/index\.\w+/gi;
+/**
+ * Normalizes `/index.{ext}` path suffixes (`/index.html`, `/index.php`, ...)
+ * to a bare trailing `/` throughout the given text, so that two otherwise
+ * identical bodies that differ only in which equivalent URL form a template
+ * happened to render (`/about/` vs `/about/index.html`) hash the same.
+ *
+ * Applied as a blanket string sweep over the whole body — not scoped to
+ * `href`/`src` attribute values — because the same trailing-suffix variance
+ * can appear anywhere a URL-shaped string is rendered as text (breadcrumbs,
+ * "print this page" widgets, JSON embedded in an inline `<script>`), and the
+ * body is treated as an opaque string for masking purposes (see
+ * `computeBodyHash`).
+ * @param body - The `<body>` inner HTML (or any text) to normalize.
+ * @returns `body` with every `/index.{ext}` suffix collapsed to `/`.
+ */
+export function normalizeUrlLikeStrings(body) {
+    return body.replaceAll(INDEX_SUFFIX_PATTERN, '/');
+}

package/lib/archive/cache/clear-archive-cache-entry.d.ts ADDED Viewed

@@ -0,0 +1,19 @@
+/**
+ * Remove a single archive cache entry (one directory previously resolved by
+ * {@link import('./resolve-archive-cache-dir.js').resolveArchiveCacheDir}).
+ *
+ * Deliberately scoped to exactly `cacheDir` — it never touches sibling
+ * entries under the same cache root (in particular, the `@nitpicker/core`
+ * analyze `table` cache is never archive-scoped and must survive a
+ * per-archive clear).
+ * @param cacheDir - Absolute path to the single cache entry to remove.
+ * @returns `true` if `cacheDir` existed and was removed, `false` if it was
+ *   already absent.
+ * @example
+ * ```ts
+ * const cacheKey = await computeArchiveCacheKey(archivePath);
+ * const cacheDir = resolveArchiveCacheDir(cacheRoot, cacheKey, archivePath);
+ * const removed = await clearArchiveCacheEntry(cacheDir);
+ * ```
+ */
+export declare function clearArchiveCacheEntry(cacheDir: string): Promise<boolean>;

package/lib/archive/cache/clear-archive-cache-entry.js ADDED Viewed

@@ -0,0 +1,25 @@
+import fs from 'node:fs/promises';
+import { pathExists } from './path-exists.js';
+/**
+ * Remove a single archive cache entry (one directory previously resolved by
+ * {@link import('./resolve-archive-cache-dir.js').resolveArchiveCacheDir}).
+ *
+ * Deliberately scoped to exactly `cacheDir` — it never touches sibling
+ * entries under the same cache root (in particular, the `@nitpicker/core`
+ * analyze `table` cache is never archive-scoped and must survive a
+ * per-archive clear).
+ * @param cacheDir - Absolute path to the single cache entry to remove.
+ * @returns `true` if `cacheDir` existed and was removed, `false` if it was
+ *   already absent.
+ * @example
+ * ```ts
+ * const cacheKey = await computeArchiveCacheKey(archivePath);
+ * const cacheDir = resolveArchiveCacheDir(cacheRoot, cacheKey, archivePath);
+ * const removed = await clearArchiveCacheEntry(cacheDir);
+ * ```
+ */
+export async function clearArchiveCacheEntry(cacheDir) {
+    const existedBefore = await pathExists(cacheDir);
+    await fs.rm(cacheDir, { recursive: true, force: true });
+    return existedBefore;
+}

package/lib/archive/cache/clear-archive-cache-root.d.ts ADDED Viewed

@@ -0,0 +1,26 @@
+/**
+ * Remove an entire archive cache root, including every tar-cache entry and
+ * any sibling directory living under it (e.g. the `@nitpicker/core` analyze
+ * `table` cache, which is resolved as a child of the same root).
+ *
+ * Pure: takes `cacheRoot` as a parameter and never resolves it itself, so
+ * tests can point it at a throwaway directory instead of the real OS temp
+ * cache. Mirrors the "manual `rm -rf` is safe" contract already documented
+ * for the tar cache (ARCHITECTURE.md) — no confirmation, no lock check.
+ *
+ * Why not lock-aware: a concurrent `extractArchiveToCache` elsewhere may be
+ * holding `<cacheDir>.lock` / writing `<cacheDir>.staging` inside this root
+ * when it is removed, which can surface as an ENOENT in that extraction (or,
+ * rarely, a second extractor racing into a freshly recreated `.staging`
+ * path). This is the same exposure a manual `rm -rf` already has today —
+ * this function does not add new risk, it just makes that pre-existing,
+ * accepted risk reachable via a single explicit command.
+ * @param cacheRoot - Absolute path to the cache root to remove.
+ * @returns `true` if `cacheRoot` existed and was removed, `false` if it was
+ *   already absent.
+ * @example
+ * ```ts
+ * const removed = await clearArchiveCacheRoot(getArchiveCacheRoot());
+ * ```
+ */
+export declare function clearArchiveCacheRoot(cacheRoot: string): Promise<boolean>;

package/lib/archive/cache/clear-archive-cache-root.js ADDED Viewed

@@ -0,0 +1,32 @@
+import fs from 'node:fs/promises';
+import { pathExists } from './path-exists.js';
+/**
+ * Remove an entire archive cache root, including every tar-cache entry and
+ * any sibling directory living under it (e.g. the `@nitpicker/core` analyze
+ * `table` cache, which is resolved as a child of the same root).
+ *
+ * Pure: takes `cacheRoot` as a parameter and never resolves it itself, so
+ * tests can point it at a throwaway directory instead of the real OS temp
+ * cache. Mirrors the "manual `rm -rf` is safe" contract already documented
+ * for the tar cache (ARCHITECTURE.md) — no confirmation, no lock check.
+ *
+ * Why not lock-aware: a concurrent `extractArchiveToCache` elsewhere may be
+ * holding `<cacheDir>.lock` / writing `<cacheDir>.staging` inside this root
+ * when it is removed, which can surface as an ENOENT in that extraction (or,
+ * rarely, a second extractor racing into a freshly recreated `.staging`
+ * path). This is the same exposure a manual `rm -rf` already has today —
+ * this function does not add new risk, it just makes that pre-existing,
+ * accepted risk reachable via a single explicit command.
+ * @param cacheRoot - Absolute path to the cache root to remove.
+ * @returns `true` if `cacheRoot` existed and was removed, `false` if it was
+ *   already absent.
+ * @example
+ * ```ts
+ * const removed = await clearArchiveCacheRoot(getArchiveCacheRoot());
+ * ```
+ */
+export async function clearArchiveCacheRoot(cacheRoot) {
+    const existedBefore = await pathExists(cacheRoot);
+    await fs.rm(cacheRoot, { recursive: true, force: true });
+    return existedBefore;
+}

package/lib/archive/cache/list-archive-cache-entries.d.ts ADDED Viewed

@@ -0,0 +1,23 @@
+import type { ArchiveCacheEntry } from './types.js';
+/**
+ * List the top-level entries found directly under an archive cache root,
+ * with a recursive size total and the most recent mtime found in each.
+ *
+ * Pure: takes `cacheRoot` as a parameter and never resolves it itself (see
+ * {@link import('./get-archive-cache-root.js').getArchiveCacheRoot} for the
+ * production root), so tests can point it at a throwaway directory instead
+ * of the real OS temp cache.
+ *
+ * Symbolic links are reported but never followed, both at the top level and
+ * while walking a directory's contents — this avoids escaping `cacheRoot`
+ * and crashing on dangling links.
+ * @param cacheRoot - Absolute path to the cache root to inspect.
+ * @returns One entry per top-level child of `cacheRoot`, in `fs.readdir`
+ *   order. Returns `[]` if `cacheRoot` does not exist.
+ * @example
+ * ```ts
+ * const entries = await listArchiveCacheEntries(getArchiveCacheRoot());
+ * const totalBytes = entries.reduce((sum, e) => sum + e.sizeBytes, 0);
+ * ```
+ */
+export declare function listArchiveCacheEntries(cacheRoot: string): Promise<ArchiveCacheEntry[]>;

package/lib/archive/cache/list-archive-cache-entries.js ADDED Viewed

@@ -0,0 +1,125 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+/**
+ * List the top-level entries found directly under an archive cache root,
+ * with a recursive size total and the most recent mtime found in each.
+ *
+ * Pure: takes `cacheRoot` as a parameter and never resolves it itself (see
+ * {@link import('./get-archive-cache-root.js').getArchiveCacheRoot} for the
+ * production root), so tests can point it at a throwaway directory instead
+ * of the real OS temp cache.
+ *
+ * Symbolic links are reported but never followed, both at the top level and
+ * while walking a directory's contents — this avoids escaping `cacheRoot`
+ * and crashing on dangling links.
+ * @param cacheRoot - Absolute path to the cache root to inspect.
+ * @returns One entry per top-level child of `cacheRoot`, in `fs.readdir`
+ *   order. Returns `[]` if `cacheRoot` does not exist.
+ * @example
+ * ```ts
+ * const entries = await listArchiveCacheEntries(getArchiveCacheRoot());
+ * const totalBytes = entries.reduce((sum, e) => sum + e.sizeBytes, 0);
+ * ```
+ */
+export async function listArchiveCacheEntries(cacheRoot) {
+    let topLevel;
+    try {
+        topLevel = await fs.readdir(cacheRoot, { withFileTypes: true });
+    }
+    catch (error) {
+        if (error.code === 'ENOENT') {
+            return [];
+        }
+        throw error;
+    }
+    const entries = [];
+    for (const dirent of topLevel) {
+        const entryPath = path.join(cacheRoot, dirent.name);
+        if (dirent.isSymbolicLink()) {
+            entries.push({
+                kind: 'unknown',
+                name: dirent.name,
+                path: entryPath,
+                sizeBytes: 0,
+                mtimeMs: 0,
+            });
+            continue;
+        }
+        if (dirent.isDirectory()) {
+            const { sizeBytes, mtimeMs } = await computeDirectoryStats(entryPath);
+            entries.push({
+                kind: classifyEntryName(dirent.name),
+                name: dirent.name,
+                path: entryPath,
+                sizeBytes,
+                mtimeMs,
+            });
+            continue;
+        }
+        const stat = await fs.stat(entryPath);
+        entries.push({
+            kind: 'unknown',
+            name: dirent.name,
+            path: entryPath,
+            sizeBytes: stat.size,
+            mtimeMs: stat.mtimeMs,
+        });
+    }
+    return entries;
+}
+/**
+ * Matches the corrupt-quarantine suffix `extractArchiveToCache` appends
+ * (`${cacheDir}.corrupt.${pid}.${counter}`, both segments always numeric —
+ * see `quarantineHalfPopulatedCacheDir` in `extract-archive-to-cache.ts`).
+ * Anchored and numeric-specific rather than a loose `includes('.corrupt.')`
+ * so an archive whose own (sanitized) basename merely contains the
+ * substring `.corrupt.` — e.g. `my.corrupt.report.nitpicker` — is not
+ * misclassified as an orphan and offered up for deletion by `cache list`.
+ */
+const CORRUPT_QUARANTINE_SUFFIX = /\.corrupt\.\d+\.\d+$/;
+/**
+ * Classify a cache-root child by name pattern alone (no filesystem access).
+ *
+ * This is a heuristic: `.staging`/corrupt-quarantine suffixes are appended
+ * by `extractArchiveToCache` onto an existing tar-cache dir name, so an
+ * archive whose own sanitized basename happens to end in exactly one of
+ * these literal suffixes is indistinguishable from a real orphan by name
+ * alone. Disambiguating fully would require re-deriving each entry's
+ * expected name from its source archive, which `list`/`clear` (by design,
+ * see grill-me scope) never has access to. Accepted as a rare, low-cost
+ * misclassification.
+ * @param name - Base name of the top-level entry.
+ */
+function classifyEntryName(name) {
+    if (name === 'table') {
+        return 'table';
+    }
+    if (name.endsWith('.staging') || CORRUPT_QUARANTINE_SUFFIX.test(name)) {
+        return 'orphan';
+    }
+    return 'tar-cache';
+}
+/**
+ * Recursively sum file sizes and find the most recent mtime under a
+ * directory. Symbolic links are neither followed nor counted.
+ * @param dirPath - Absolute path to the directory to walk.
+ */
+async function computeDirectoryStats(dirPath) {
+    let sizeBytes = 0;
+    let mtimeMs = 0;
+    const children = await fs.readdir(dirPath, { recursive: true, withFileTypes: true });
+    for (const child of children) {
+        if (!child.isFile()) {
+            continue;
+        }
+        const childPath = path.join(child.parentPath, child.name);
+        const stat = await fs.stat(childPath);
+        sizeBytes += stat.size;
+        mtimeMs = Math.max(mtimeMs, stat.mtimeMs);
+    }
+    if (mtimeMs === 0) {
+        const dirStat = await fs.stat(dirPath);
+        mtimeMs = dirStat.mtimeMs;
+    }
+    return { sizeBytes, mtimeMs };
+}

package/lib/archive/cache/path-exists.d.ts ADDED Viewed

@@ -0,0 +1,9 @@
+/**
+ * Async existence probe — avoids blocking the event loop on the common
+ * "check before remove" path shared by
+ * {@link import('./clear-archive-cache-root.js').clearArchiveCacheRoot} and
+ * {@link import('./clear-archive-cache-entry.js').clearArchiveCacheEntry}.
+ * @param targetPath - Absolute path to probe.
+ * @returns `true` if the path is reachable via `fs.access`.
+ */
+export declare function pathExists(targetPath: string): Promise<boolean>;

package/lib/archive/cache/path-exists.js ADDED Viewed

@@ -0,0 +1,18 @@
+import fs from 'node:fs/promises';
+/**
+ * Async existence probe — avoids blocking the event loop on the common
+ * "check before remove" path shared by
+ * {@link import('./clear-archive-cache-root.js').clearArchiveCacheRoot} and
+ * {@link import('./clear-archive-cache-entry.js').clearArchiveCacheEntry}.
+ * @param targetPath - Absolute path to probe.
+ * @returns `true` if the path is reachable via `fs.access`.
+ */
+export async function pathExists(targetPath) {
+    try {
+        await fs.access(targetPath);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}

package/lib/archive/cache/types.d.ts ADDED Viewed

@@ -0,0 +1,33 @@
+/**
+ * Classification of a top-level entry found directly under an archive cache
+ * root ({@link import('./get-archive-cache-root.js').getArchiveCacheRoot}).
+ *
+ * - `'tar-cache'`: a per-archive extraction produced by
+ *   {@link import('./resolve-archive-cache-dir.js').resolveArchiveCacheDir}
+ *   (`<cacheKey>-<safeBasename>`).
+ * - `'table'`: the `@nitpicker/core` analyze-plugin scratch cache
+ *   (`getTableCacheRoot()`), a sibling directory literally named `table`.
+ * - `'orphan'`: a `.staging` or `.corrupt.<pid>.<n>` leftover from an
+ *   interrupted {@link import('./extract-archive-to-cache.js').extractArchiveToCache}
+ *   run (see that file's quarantine/staging logic).
+ * - `'unknown'`: anything else (a stray file, a foreign directory a user
+ *   dropped into the cache root).
+ *
+ * `'tar-cache'` and `'table'` cannot collide: `resolveArchiveCacheDir` always
+ * prefixes the cache key (digits and hyphens), so a tar-cache directory name
+ * is never the bare literal `table`.
+ */
+export type ArchiveCacheEntryKind = 'tar-cache' | 'table' | 'orphan' | 'unknown';
+/** A single top-level entry found under an archive cache root. */
+export interface ArchiveCacheEntry {
+    /** How this entry was classified by name pattern. */
+    readonly kind: ArchiveCacheEntryKind;
+    /** Base name of the entry (its final path segment). */
+    readonly name: string;
+    /** Absolute path to the entry. */
+    readonly path: string;
+    /** Total size in bytes of all regular files found under this entry (recursive for directories). */
+    readonly sizeBytes: number;
+    /** Most recent mtime (epoch ms) found among the entry's files, or the entry's own mtime if it has none. */
+    readonly mtimeMs: number;
+}

package/lib/archive/cache/types.js ADDED Viewed

	@@ -0,0 +1 @@
1	+ export {};

package/lib/archive/create-adjunct-tables.d.ts CHANGED Viewed

@@ -14,13 +14,27 @@ import type { Knex } from 'knex';
  *   `MainContentsData` sub-entity arrays, one row per DOM element, FK →
  *   `content_items(id)`
  * - `inventory_runs` — `--inventory` audit log (no FK; append-only)
+ * - `network_outages` — operator-network-outage journal (no FK; append-only
+ *   except `ended_at`, which is written once on recovery)
+ * - `dedupe_cap_events` — `--dedupe-cap` same-cluster soft-cap audit log (no
+ *   FK; append-only except `rejected_count`, which is written once at
+ *   `crawlEnd`)
  * - `analysis_text_refs` + `analysis_violations` — analyze-phase findings,
  *   FK → `content_items(id)`
  * - `page_templates` — DOM-structure template classification (`--templates`,
  *   `@nitpicker/core`'s `template-classification/`), one row per classified
  *   page, FK → `content_items(id)`
+ * - `page_template_clusters` — one row per distinct `page_templates.template_key`,
+ *   holding `@d-zero/page-cluster`'s cluster-selection evidence (no FK;
+ *   `template_key` is not a `page_templates` FK target, so consistency is
+ *   maintained by replacing both tables together, not by a foreign key)
  * - `page_html_blobs` + `page_html_ref` — content-addressable HTML
  *   snapshots, FK → `content_items(id)`
+ * - `console_log_items` — content-addressable dictionary of distinct
+ *   console messages / page errors (no FK; hash-deduplicated across every
+ *   page in the archive, mirroring `text_refs` / `json_refs`)
+ * - `page_console_logs` — one row per (page, console log) occurrence, FK →
+ *   `content_items(id)` and `console_log_items(id)`
  *
  * The DDL is shared between fresh-archive provisioning ({@link initSchema}
  * calls this right after `createEntityTables`) and the migration script

package/lib/archive/create-adjunct-tables.js CHANGED Viewed

@@ -13,13 +13,27 @@
  *   `MainContentsData` sub-entity arrays, one row per DOM element, FK →
  *   `content_items(id)`
  * - `inventory_runs` — `--inventory` audit log (no FK; append-only)
+ * - `network_outages` — operator-network-outage journal (no FK; append-only
+ *   except `ended_at`, which is written once on recovery)
+ * - `dedupe_cap_events` — `--dedupe-cap` same-cluster soft-cap audit log (no
+ *   FK; append-only except `rejected_count`, which is written once at
+ *   `crawlEnd`)
  * - `analysis_text_refs` + `analysis_violations` — analyze-phase findings,
  *   FK → `content_items(id)`
  * - `page_templates` — DOM-structure template classification (`--templates`,
  *   `@nitpicker/core`'s `template-classification/`), one row per classified
  *   page, FK → `content_items(id)`
+ * - `page_template_clusters` — one row per distinct `page_templates.template_key`,
+ *   holding `@d-zero/page-cluster`'s cluster-selection evidence (no FK;
+ *   `template_key` is not a `page_templates` FK target, so consistency is
+ *   maintained by replacing both tables together, not by a foreign key)
  * - `page_html_blobs` + `page_html_ref` — content-addressable HTML
  *   snapshots, FK → `content_items(id)`
+ * - `console_log_items` — content-addressable dictionary of distinct
+ *   console messages / page errors (no FK; hash-deduplicated across every
+ *   page in the archive, mirroring `text_refs` / `json_refs`)
+ * - `page_console_logs` — one row per (page, console log) occurrence, FK →
+ *   `content_items(id)` and `console_log_items(id)`
  *
  * The DDL is shared between fresh-archive provisioning ({@link initSchema}
  * calls this right after `createEntityTables`) and the migration script
@@ -279,10 +293,81 @@ export async function createAdjunctTables(instance) {
             t.integer('new_pages').nullable();
             t.integer('new_resources').nullable();
             t.integer('scope_skipped').nullable();
+            t.integer('invalid_skipped').nullable();
             t.text('notes').nullable();
             t.index('ran_at');
         });
     }
+    if (!(await instance.schema.hasTable('network_outages'))) {
+        await instance.schema.createTable('network_outages', (t) => {
+            // One row per detected operator-network outage. The archive's
+            // evidence that a run of `dns`/`local-network`-shaped failures
+            // was caused by the crawl operator's own connectivity, not the
+            // target sites — used to retroactively re-classify `crawl_errors`
+            // / `page_errors` rows whose `createdAt` falls inside a window
+            // (see `is-within-outage-window.ts`). No index: a crawl session
+            // produces at most a handful of these rows, so a full in-memory
+            // scan by every consumer is cheaper than maintaining a B-tree
+            // that few queries would use (`ARCHITECTURE.md`'s
+            // perf-index-is-not-free / evidence-before-indexing rules).
+            t.increments('id');
+            // Backdated to the earliest error still inside the detector's
+            // sliding window at trigger time, NOT the trigger instant —
+            // see `NetworkOutageDetector`'s `OutageSuspect.startedAt`.
+            t.integer('started_at').notNullable();
+            // When the sliding window actually crossed both thresholds.
+            t.integer('detected_at').notNullable();
+            // NULL until a recovery probe succeeds. A row can also be left
+            // NULL forever if the crawl process is killed mid-outage; readers
+            // MUST NOT treat a NULL `ended_at` as an unbounded window (that
+            // would retroactively cover every later error as
+            // network-caused) — see `is-within-outage-window.ts` and the
+            // writer-side clamp-on-next-open in `db-ops/outages/`.
+            t.integer('ended_at').nullable();
+            t.string('probe_host').nullable();
+            t.integer('trigger_error_count').notNullable();
+            t.integer('trigger_host_count').notNullable();
+        });
+    }
+    if (!(await instance.schema.hasTable('dedupe_cap_events'))) {
+        await instance.schema.createTable('dedupe_cap_events', (t) => {
+            // One row per URL shape the `--dedupe-cap` same-cluster soft cap
+            // (`DedupeCapTracker`) confirmed as a trap during this crawl. No
+            // index: a crawl produces at most a handful of these rows (same
+            // reasoning as `network_outages`, above).
+            t.increments('id');
+            // The URL shape key (`computeShapeKey`) that capped — a template
+            // with placeholders (e.g. `example.com/news/date/{n}/`), not a
+            // literal URL.
+            t.string('shape_key').notNullable();
+            // One concrete URL matching this shape, captured at cap time so a
+            // human reading the audit log can identify what was being
+            // crawled — `shape_key` alone is a template, not a navigable URL.
+            t.string('sample_url').notNullable();
+            // `computeBodyHash` result recorded at cap time. Nullable only in
+            // the sense that BLOB columns are nullable by default; every row
+            // this feature writes populates it (a page with no rendered
+            // `<body>` never reaches the tracker — see `Crawler#handleResult`).
+            t.binary('body_hash').nullable();
+            // The Misra-Gries threshold that actually triggered the cap,
+            // after halving for the `body_hash`-match / `og:url`-mismatch
+            // confidence signals — NOT necessarily equal to `--dedupe-cap`'s
+            // raw value.
+            t.integer('effective_threshold').notNullable();
+            // The tracker's Misra-Gries counter value at cap time: a LOWER
+            // BOUND on the number of matching-signature pages seen for this
+            // shape, not an exact observation count (see `DedupeCapTracker`).
+            t.integer('observed_count').notNullable();
+            t.integer('detected_at').notNullable();
+            // NULL until `crawlEnd` finalizes it (see
+            // `Crawler#getDedupeCapRejections`). Unlike `network_outages.ended_at`,
+            // a NULL here has no ambiguous "still ongoing" reading — a
+            // crawl that never reached `crawlEnd` simply left the count
+            // undetermined, so no boot-time reconciliation pass is needed
+            // (readers display "unknown", not "0" or "unbounded").
+            t.integer('rejected_count').nullable();
+        });
+    }
     if (!(await instance.schema.hasTable('analysis_text_refs'))) {
         await instance.raw(`
 			CREATE TABLE analysis_text_refs (
@@ -334,6 +419,34 @@ export async function createAdjunctTables(instance) {
 			) WITHOUT ROWID
 		`);
     }
+    // One row per distinct `template_key` produced by the same `--templates`
+    // classification run, holding `@d-zero/page-cluster`'s cluster-selection
+    // evidence (`ClusterReason`, renamed `TemplateClusterReason` on this side)
+    // as a zstd-compressed JSON blob — same BLOB+codec+size shape as
+    // `page_html_blobs` below. A column on `page_templates` was rejected: that
+    // table is one row per *page*, so the same cluster's reason would be
+    // duplicated across every member page (multi-GB on a large archive with a
+    // few-hundred-member cluster). A `json_refs` row was also rejected: reason
+    // payloads differ per cluster (distinct `memberCount`/token sets), so
+    // content-address dedup would not pay for itself, and `json_refs` is a
+    // shared dictionary that other tables reference — this table's full
+    // replace-on-every-run write pattern (see `replacePageTemplates`) would
+    // otherwise leave orphaned rows behind with no owner able to delete them.
+    // No FK to `page_templates`: `template_key` is not that table's primary
+    // key (`page_id` is), so there is nothing to reference — consistency is
+    // instead maintained by replacing both tables in the same transaction.
+    if (!(await instance.schema.hasTable('page_template_clusters'))) {
+        await instance.raw(`
+			CREATE TABLE page_template_clusters (
+				template_key TEXT PRIMARY KEY,
+				member_count INTEGER NOT NULL,
+				reason_json  BLOB NOT NULL,
+				codec        TEXT NOT NULL CHECK(codec IN ('zstd', 'none')),
+				size_raw     INTEGER NOT NULL,
+				size_stored  INTEGER NOT NULL
+			) WITHOUT ROWID
+		`);
+    }
     // Content-addressable HTML blob storage. Knex's schema builder doesn't
     // expose a WITHOUT ROWID toggle, so the BLOB tables are created via raw
     // SQL. WITHOUT ROWID keeps the rows packed inside the b-tree leaves
@@ -360,4 +473,60 @@ export async function createAdjunctTables(instance) {
 		`);
         await instance.raw('CREATE INDEX idx_page_html_ref_hash ON page_html_ref(hash)');
     }
+    // Content-addressable dictionary of distinct console messages / page
+    // errors (beholder's `ConsoleLogEntry`, issue #228). `hash` is
+    // SHA-256 over a canonical tuple of every content field (type, text,
+    // args JSON, location, stack) — the same identical warning emitted by
+    // a shared framework on every page therefore collapses to one row
+    // regardless of how many pages or how many times it fires, mirroring
+    // `text_refs` / `json_refs`. `args_json_id` is nullable because a
+    // call with no arguments (or one whose args failed to
+    // `JSON.stringify`, e.g. a circular reference) has nothing to store.
+    // `text_id` is nullable too: `text_refs` never stores the empty
+    // string (its dictionary upsert treats `''` as "nothing to dedupe"),
+    // so a call like `console.log()` with zero arguments — whose
+    // `text` beholder reports as `''` — has no `text_refs` row to point
+    // at; `text_id = NULL` there means "empty text", read back as `''`.
+    // `type` keeps its own index for the Console Logs view's type filter.
+    if (!(await instance.schema.hasTable('console_log_items'))) {
+        await instance.raw(`
+			CREATE TABLE console_log_items (
+				id             INTEGER PRIMARY KEY,
+				hash           BLOB NOT NULL UNIQUE,
+				type           TEXT NOT NULL,
+				text_id        INTEGER REFERENCES text_refs(id),
+				args_json_id   INTEGER REFERENCES json_refs(id),
+				loc_url_id     INTEGER REFERENCES url_refs(id),
+				loc_line       INTEGER,
+				loc_column     INTEGER,
+				stack_text_id  INTEGER REFERENCES text_refs(id)
+			)
+		`);
+        await instance.raw('CREATE INDEX idx_console_log_items_type ON console_log_items(type)');
+    }
+    // One row per (page, console log) occurrence — beholder captures a
+    // `ts` per firing, so the same message logged 3 times on one page
+    // yields 3 rows (unlike `anchor_edges`' first-wins dedup: an
+    // occurrence count matters here, not just presence). Replaced
+    // wholesale per page on every non-empty scrape by
+    // `replaceConsoleLogs`, the same Scoped-Replace pattern as
+    // `anchor_edges` / `image_items` — there is no natural key to UPDATE
+    // a specific prior occurrence against.
+    if (!(await instance.schema.hasTable('page_console_logs'))) {
+        await instance.schema.createTable('page_console_logs', (t) => {
+            t.increments('id');
+            t.integer('pageId')
+                .notNullable()
+                .unsigned()
+                .references('content_items.id')
+                .onDelete('CASCADE');
+            t.integer('consoleLogId')
+                .notNullable()
+                .unsigned()
+                .references('console_log_items.id');
+            t.integer('ts').notNullable();
+            t.index('pageId');
+            t.index('consoleLogId');
+        });
+    }
 }

package/lib/archive/create-entity-tables.js CHANGED Viewed

@@ -197,6 +197,7 @@ export async function createEntityTables(instance) {
 			content_length   INTEGER,
 			header_set_id    INTEGER REFERENCES header_sets(id),
 			redirect_dest_id INTEGER REFERENCES content_items(id) DEFERRABLE INITIALLY DEFERRED,
+			alias_of_id      INTEGER REFERENCES content_items(id) DEFERRABLE INITIALLY DEFERRED,
 			source           TEXT NOT NULL DEFAULT 'crawled',
 			first_crawled_at INTEGER,
 			last_crawled_at  INTEGER,
@@ -208,6 +209,15 @@ export async function createEntityTables(instance) {
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_external ON content_items(is_external)');
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_scraped ON content_items(scraped)');
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_redirect_dest_id ON content_items(redirect_dest_id)');
+    // `idx_content_items_alias_of_id` is NOT created here even though
+    // `alias_of_id` is: this DDL runs unconditionally on every archive open
+    // (including legacy archives that still lack the column at this point,
+    // before `migrateContentItemsAliasOfId` adds it), so an unconditional
+    // `CREATE INDEX ... alias_of_id` here would fail with `no such column`
+    // on any archive that predates this feature. The index is created in
+    // `migrateContentItemsAliasOfId` instead, which runs after the
+    // column-add guard for both fresh and legacy archives (same reasoning as
+    // `page_meta.body_hash`'s index).
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_content_type_id ON content_items(content_type_id)');
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_crawl_order ON content_items(crawl_order)');
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_source ON content_items(source)');
@@ -279,10 +289,20 @@ export async function createEntityTables(instance) {
 			main_content_audio_count    INTEGER,
 			main_content_canvas_count   INTEGER,
 			scroll_height_desktop       INTEGER,
-			scroll_height_mobile        INTEGER
+			scroll_height_mobile        INTEGER,
+			body_hash                   BLOB,
+			console_error_count         INTEGER
 		)
 	`);
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_page_meta_og_type ON page_meta(og_type)');
+    // `idx_page_meta_body_hash` is NOT created here even though `body_hash`
+    // is: this DDL runs unconditionally on every archive open (including
+    // legacy archives that still lack the column at this point, before
+    // `migratePageMetaBodyHash` adds it), so an unconditional
+    // `CREATE INDEX ... body_hash` here would fail with `no such column` on
+    // any archive that predates this feature. The index is created in
+    // `migratePageMetaBodyHash` instead, which runs after the column-add
+    // guard for both fresh and legacy archives.
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_page_meta_robots_noindex ON page_meta(robots_noindex)');
     await instance.raw(`
 		CREATE TABLE IF NOT EXISTS resource_items (