@nitpicker/crawler 0.14.0 → 0.15.0

package/lib/archive/cache/clear-archive-cache-root.js

@@ -0,0 +1,32 @@
+import fs from 'node:fs/promises';
+import { pathExists } from './path-exists.js';
+/**
+ * Remove an entire archive cache root, including every tar-cache entry and
+ * any sibling directory living under it (e.g. the `@nitpicker/core` analyze
+ * `table` cache, which is resolved as a child of the same root).
+ *
+ * Pure: takes `cacheRoot` as a parameter and never resolves it itself, so
+ * tests can point it at a throwaway directory instead of the real OS temp
+ * cache. Mirrors the "manual `rm -rf` is safe" contract already documented
+ * for the tar cache (ARCHITECTURE.md) — no confirmation, no lock check.
+ *
+ * Why not lock-aware: a concurrent `extractArchiveToCache` elsewhere may be
+ * holding `<cacheDir>.lock` / writing `<cacheDir>.staging` inside this root
+ * when it is removed, which can surface as an ENOENT in that extraction (or,
+ * rarely, a second extractor racing into a freshly recreated `.staging`
+ * path). This is the same exposure a manual `rm -rf` already has today —
+ * this function does not add new risk, it just makes that pre-existing,
+ * accepted risk reachable via a single explicit command.
+ * @param cacheRoot - Absolute path to the cache root to remove.
+ * @returns `true` if `cacheRoot` existed and was removed, `false` if it was
+ *   already absent.
+ * @example
+ * ```ts
+ * const removed = await clearArchiveCacheRoot(getArchiveCacheRoot());
+ * ```
+ */
+export async function clearArchiveCacheRoot(cacheRoot) {
+    const existedBefore = await pathExists(cacheRoot);
+    await fs.rm(cacheRoot, { recursive: true, force: true });
+    return existedBefore;
+}

package/lib/archive/cache/list-archive-cache-entries.d.ts

@@ -0,0 +1,23 @@
+import type { ArchiveCacheEntry } from './types.js';
+/**
+ * List the top-level entries found directly under an archive cache root,
+ * with a recursive size total and the most recent mtime found in each.
+ *
+ * Pure: takes `cacheRoot` as a parameter and never resolves it itself (see
+ * {@link import('./get-archive-cache-root.js').getArchiveCacheRoot} for the
+ * production root), so tests can point it at a throwaway directory instead
+ * of the real OS temp cache.
+ *
+ * Symbolic links are reported but never followed, both at the top level and
+ * while walking a directory's contents — this avoids escaping `cacheRoot`
+ * and crashing on dangling links.
+ * @param cacheRoot - Absolute path to the cache root to inspect.
+ * @returns One entry per top-level child of `cacheRoot`, in `fs.readdir`
+ *   order. Returns `[]` if `cacheRoot` does not exist.
+ * @example
+ * ```ts
+ * const entries = await listArchiveCacheEntries(getArchiveCacheRoot());
+ * const totalBytes = entries.reduce((sum, e) => sum + e.sizeBytes, 0);
+ * ```
+ */
+export declare function listArchiveCacheEntries(cacheRoot: string): Promise<ArchiveCacheEntry[]>;

package/lib/archive/cache/list-archive-cache-entries.js

@@ -0,0 +1,125 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+/**
+ * List the top-level entries found directly under an archive cache root,
+ * with a recursive size total and the most recent mtime found in each.
+ *
+ * Pure: takes `cacheRoot` as a parameter and never resolves it itself (see
+ * {@link import('./get-archive-cache-root.js').getArchiveCacheRoot} for the
+ * production root), so tests can point it at a throwaway directory instead
+ * of the real OS temp cache.
+ *
+ * Symbolic links are reported but never followed, both at the top level and
+ * while walking a directory's contents — this avoids escaping `cacheRoot`
+ * and crashing on dangling links.
+ * @param cacheRoot - Absolute path to the cache root to inspect.
+ * @returns One entry per top-level child of `cacheRoot`, in `fs.readdir`
+ *   order. Returns `[]` if `cacheRoot` does not exist.
+ * @example
+ * ```ts
+ * const entries = await listArchiveCacheEntries(getArchiveCacheRoot());
+ * const totalBytes = entries.reduce((sum, e) => sum + e.sizeBytes, 0);
+ * ```
+ */
+export async function listArchiveCacheEntries(cacheRoot) {
+    let topLevel;
+    try {
+        topLevel = await fs.readdir(cacheRoot, { withFileTypes: true });
+    }
+    catch (error) {
+        if (error.code === 'ENOENT') {
+            return [];
+        }
+        throw error;
+    }
+    const entries = [];
+    for (const dirent of topLevel) {
+        const entryPath = path.join(cacheRoot, dirent.name);
+        if (dirent.isSymbolicLink()) {
+            entries.push({
+                kind: 'unknown',
+                name: dirent.name,
+                path: entryPath,
+                sizeBytes: 0,
+                mtimeMs: 0,
+            });
+            continue;
+        }
+        if (dirent.isDirectory()) {
+            const { sizeBytes, mtimeMs } = await computeDirectoryStats(entryPath);
+            entries.push({
+                kind: classifyEntryName(dirent.name),
+                name: dirent.name,
+                path: entryPath,
+                sizeBytes,
+                mtimeMs,
+            });
+            continue;
+        }
+        const stat = await fs.stat(entryPath);
+        entries.push({
+            kind: 'unknown',
+            name: dirent.name,
+            path: entryPath,
+            sizeBytes: stat.size,
+            mtimeMs: stat.mtimeMs,
+        });
+    }
+    return entries;
+}
+/**
+ * Matches the corrupt-quarantine suffix `extractArchiveToCache` appends
+ * (`${cacheDir}.corrupt.${pid}.${counter}`, both segments always numeric —
+ * see `quarantineHalfPopulatedCacheDir` in `extract-archive-to-cache.ts`).
+ * Anchored and numeric-specific rather than a loose `includes('.corrupt.')`
+ * so an archive whose own (sanitized) basename merely contains the
+ * substring `.corrupt.` — e.g. `my.corrupt.report.nitpicker` — is not
+ * misclassified as an orphan and offered up for deletion by `cache list`.
+ */
+const CORRUPT_QUARANTINE_SUFFIX = /\.corrupt\.\d+\.\d+$/;
+/**
+ * Classify a cache-root child by name pattern alone (no filesystem access).
+ *
+ * This is a heuristic: `.staging`/corrupt-quarantine suffixes are appended
+ * by `extractArchiveToCache` onto an existing tar-cache dir name, so an
+ * archive whose own sanitized basename happens to end in exactly one of
+ * these literal suffixes is indistinguishable from a real orphan by name
+ * alone. Disambiguating fully would require re-deriving each entry's
+ * expected name from its source archive, which `list`/`clear` (by design,
+ * see grill-me scope) never has access to. Accepted as a rare, low-cost
+ * misclassification.
+ * @param name - Base name of the top-level entry.
+ */
+function classifyEntryName(name) {
+    if (name === 'table') {
+        return 'table';
+    }
+    if (name.endsWith('.staging') || CORRUPT_QUARANTINE_SUFFIX.test(name)) {
+        return 'orphan';
+    }
+    return 'tar-cache';
+}
+/**
+ * Recursively sum file sizes and find the most recent mtime under a
+ * directory. Symbolic links are neither followed nor counted.
+ * @param dirPath - Absolute path to the directory to walk.
+ */
+async function computeDirectoryStats(dirPath) {
+    let sizeBytes = 0;
+    let mtimeMs = 0;
+    const children = await fs.readdir(dirPath, { recursive: true, withFileTypes: true });
+    for (const child of children) {
+        if (!child.isFile()) {
+            continue;
+        }
+        const childPath = path.join(child.parentPath, child.name);
+        const stat = await fs.stat(childPath);
+        sizeBytes += stat.size;
+        mtimeMs = Math.max(mtimeMs, stat.mtimeMs);
+    }
+    if (mtimeMs === 0) {
+        const dirStat = await fs.stat(dirPath);
+        mtimeMs = dirStat.mtimeMs;
+    }
+    return { sizeBytes, mtimeMs };
+}

package/lib/archive/cache/path-exists.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Async existence probe — avoids blocking the event loop on the common
+ * "check before remove" path shared by
+ * {@link import('./clear-archive-cache-root.js').clearArchiveCacheRoot} and
+ * {@link import('./clear-archive-cache-entry.js').clearArchiveCacheEntry}.
+ * @param targetPath - Absolute path to probe.
+ * @returns `true` if the path is reachable via `fs.access`.
+ */
+export declare function pathExists(targetPath: string): Promise<boolean>;

package/lib/archive/cache/path-exists.js

@@ -0,0 +1,18 @@
+import fs from 'node:fs/promises';
+/**
+ * Async existence probe — avoids blocking the event loop on the common
+ * "check before remove" path shared by
+ * {@link import('./clear-archive-cache-root.js').clearArchiveCacheRoot} and
+ * {@link import('./clear-archive-cache-entry.js').clearArchiveCacheEntry}.
+ * @param targetPath - Absolute path to probe.
+ * @returns `true` if the path is reachable via `fs.access`.
+ */
+export async function pathExists(targetPath) {
+    try {
+        await fs.access(targetPath);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}

package/lib/archive/cache/types.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Classification of a top-level entry found directly under an archive cache
+ * root ({@link import('./get-archive-cache-root.js').getArchiveCacheRoot}).
+ *
+ * - `'tar-cache'`: a per-archive extraction produced by
+ *   {@link import('./resolve-archive-cache-dir.js').resolveArchiveCacheDir}
+ *   (`<cacheKey>-<safeBasename>`).
+ * - `'table'`: the `@nitpicker/core` analyze-plugin scratch cache
+ *   (`getTableCacheRoot()`), a sibling directory literally named `table`.
+ * - `'orphan'`: a `.staging` or `.corrupt.<pid>.<n>` leftover from an
+ *   interrupted {@link import('./extract-archive-to-cache.js').extractArchiveToCache}
+ *   run (see that file's quarantine/staging logic).
+ * - `'unknown'`: anything else (a stray file, a foreign directory a user
+ *   dropped into the cache root).
+ *
+ * `'tar-cache'` and `'table'` cannot collide: `resolveArchiveCacheDir` always
+ * prefixes the cache key (digits and hyphens), so a tar-cache directory name
+ * is never the bare literal `table`.
+ */
+export type ArchiveCacheEntryKind = 'tar-cache' | 'table' | 'orphan' | 'unknown';
+/** A single top-level entry found under an archive cache root. */
+export interface ArchiveCacheEntry {
+    /** How this entry was classified by name pattern. */
+    readonly kind: ArchiveCacheEntryKind;
+    /** Base name of the entry (its final path segment). */
+    readonly name: string;
+    /** Absolute path to the entry. */
+    readonly path: string;
+    /** Total size in bytes of all regular files found under this entry (recursive for directories). */
+    readonly sizeBytes: number;
+    /** Most recent mtime (epoch ms) found among the entry's files, or the entry's own mtime if it has none. */
+    readonly mtimeMs: number;
+}

package/lib/archive/cache/types.js

@@ -0,0 +1 @@
+export {};

package/lib/archive/create-adjunct-tables.d.ts

@@ -14,13 +14,24 @@ import type { Knex } from 'knex';
  *   `MainContentsData` sub-entity arrays, one row per DOM element, FK →
  *   `content_items(id)`
  * - `inventory_runs` — `--inventory` audit log (no FK; append-only)
+ * - `network_outages` — operator-network-outage journal (no FK; append-only
+ *   except `ended_at`, which is written once on recovery)
  * - `analysis_text_refs` + `analysis_violations` — analyze-phase findings,
  *   FK → `content_items(id)`
  * - `page_templates` — DOM-structure template classification (`--templates`,
  *   `@nitpicker/core`'s `template-classification/`), one row per classified
  *   page, FK → `content_items(id)`
+ * - `page_template_clusters` — one row per distinct `page_templates.template_key`,
+ *   holding `@d-zero/page-cluster`'s cluster-selection evidence (no FK;
+ *   `template_key` is not a `page_templates` FK target, so consistency is
+ *   maintained by replacing both tables together, not by a foreign key)
  * - `page_html_blobs` + `page_html_ref` — content-addressable HTML
  *   snapshots, FK → `content_items(id)`
+ * - `console_log_items` — content-addressable dictionary of distinct
+ *   console messages / page errors (no FK; hash-deduplicated across every
+ *   page in the archive, mirroring `text_refs` / `json_refs`)
+ * - `page_console_logs` — one row per (page, console log) occurrence, FK →
+ *   `content_items(id)` and `console_log_items(id)`
  *
  * The DDL is shared between fresh-archive provisioning ({@link initSchema}
  * calls this right after `createEntityTables`) and the migration script

package/lib/archive/create-adjunct-tables.js

@@ -13,13 +13,24 @@
  *   `MainContentsData` sub-entity arrays, one row per DOM element, FK →
  *   `content_items(id)`
  * - `inventory_runs` — `--inventory` audit log (no FK; append-only)
+ * - `network_outages` — operator-network-outage journal (no FK; append-only
+ *   except `ended_at`, which is written once on recovery)
  * - `analysis_text_refs` + `analysis_violations` — analyze-phase findings,
  *   FK → `content_items(id)`
  * - `page_templates` — DOM-structure template classification (`--templates`,
  *   `@nitpicker/core`'s `template-classification/`), one row per classified
  *   page, FK → `content_items(id)`
+ * - `page_template_clusters` — one row per distinct `page_templates.template_key`,
+ *   holding `@d-zero/page-cluster`'s cluster-selection evidence (no FK;
+ *   `template_key` is not a `page_templates` FK target, so consistency is
+ *   maintained by replacing both tables together, not by a foreign key)
  * - `page_html_blobs` + `page_html_ref` — content-addressable HTML
  *   snapshots, FK → `content_items(id)`
+ * - `console_log_items` — content-addressable dictionary of distinct
+ *   console messages / page errors (no FK; hash-deduplicated across every
+ *   page in the archive, mirroring `text_refs` / `json_refs`)
+ * - `page_console_logs` — one row per (page, console log) occurrence, FK →
+ *   `content_items(id)` and `console_log_items(id)`
  *
  * The DDL is shared between fresh-archive provisioning ({@link initSchema}
  * calls this right after `createEntityTables`) and the migration script
@@ -279,10 +290,42 @@ export async function createAdjunctTables(instance) {
             t.integer('new_pages').nullable();
             t.integer('new_resources').nullable();
             t.integer('scope_skipped').nullable();
+            t.integer('invalid_skipped').nullable();
             t.text('notes').nullable();
             t.index('ran_at');
         });
     }
+    if (!(await instance.schema.hasTable('network_outages'))) {
+        await instance.schema.createTable('network_outages', (t) => {
+            // One row per detected operator-network outage. The archive's
+            // evidence that a run of `dns`/`local-network`-shaped failures
+            // was caused by the crawl operator's own connectivity, not the
+            // target sites — used to retroactively re-classify `crawl_errors`
+            // / `page_errors` rows whose `createdAt` falls inside a window
+            // (see `is-within-outage-window.ts`). No index: a crawl session
+            // produces at most a handful of these rows, so a full in-memory
+            // scan by every consumer is cheaper than maintaining a B-tree
+            // that few queries would use (`ARCHITECTURE.md`'s
+            // perf-index-is-not-free / evidence-before-indexing rules).
+            t.increments('id');
+            // Backdated to the earliest error still inside the detector's
+            // sliding window at trigger time, NOT the trigger instant —
+            // see `NetworkOutageDetector`'s `OutageSuspect.startedAt`.
+            t.integer('started_at').notNullable();
+            // When the sliding window actually crossed both thresholds.
+            t.integer('detected_at').notNullable();
+            // NULL until a recovery probe succeeds. A row can also be left
+            // NULL forever if the crawl process is killed mid-outage; readers
+            // MUST NOT treat a NULL `ended_at` as an unbounded window (that
+            // would retroactively cover every later error as
+            // network-caused) — see `is-within-outage-window.ts` and the
+            // writer-side clamp-on-next-open in `db-ops/outages/`.
+            t.integer('ended_at').nullable();
+            t.string('probe_host').nullable();
+            t.integer('trigger_error_count').notNullable();
+            t.integer('trigger_host_count').notNullable();
+        });
+    }
     if (!(await instance.schema.hasTable('analysis_text_refs'))) {
         await instance.raw(`
 			CREATE TABLE analysis_text_refs (
@@ -334,6 +377,34 @@ export async function createAdjunctTables(instance) {
 			) WITHOUT ROWID
 		`);
     }
+    // One row per distinct `template_key` produced by the same `--templates`
+    // classification run, holding `@d-zero/page-cluster`'s cluster-selection
+    // evidence (`ClusterReason`, renamed `TemplateClusterReason` on this side)
+    // as a zstd-compressed JSON blob — same BLOB+codec+size shape as
+    // `page_html_blobs` below. A column on `page_templates` was rejected: that
+    // table is one row per *page*, so the same cluster's reason would be
+    // duplicated across every member page (multi-GB on a large archive with a
+    // few-hundred-member cluster). A `json_refs` row was also rejected: reason
+    // payloads differ per cluster (distinct `memberCount`/token sets), so
+    // content-address dedup would not pay for itself, and `json_refs` is a
+    // shared dictionary that other tables reference — this table's full
+    // replace-on-every-run write pattern (see `replacePageTemplates`) would
+    // otherwise leave orphaned rows behind with no owner able to delete them.
+    // No FK to `page_templates`: `template_key` is not that table's primary
+    // key (`page_id` is), so there is nothing to reference — consistency is
+    // instead maintained by replacing both tables in the same transaction.
+    if (!(await instance.schema.hasTable('page_template_clusters'))) {
+        await instance.raw(`
+			CREATE TABLE page_template_clusters (
+				template_key TEXT PRIMARY KEY,
+				member_count INTEGER NOT NULL,
+				reason_json  BLOB NOT NULL,
+				codec        TEXT NOT NULL CHECK(codec IN ('zstd', 'none')),
+				size_raw     INTEGER NOT NULL,
+				size_stored  INTEGER NOT NULL
+			) WITHOUT ROWID
+		`);
+    }
     // Content-addressable HTML blob storage. Knex's schema builder doesn't
     // expose a WITHOUT ROWID toggle, so the BLOB tables are created via raw
     // SQL. WITHOUT ROWID keeps the rows packed inside the b-tree leaves
@@ -360,4 +431,60 @@ export async function createAdjunctTables(instance) {
 		`);
         await instance.raw('CREATE INDEX idx_page_html_ref_hash ON page_html_ref(hash)');
     }
+    // Content-addressable dictionary of distinct console messages / page
+    // errors (beholder's `ConsoleLogEntry`, issue #228). `hash` is
+    // SHA-256 over a canonical tuple of every content field (type, text,
+    // args JSON, location, stack) — the same identical warning emitted by
+    // a shared framework on every page therefore collapses to one row
+    // regardless of how many pages or how many times it fires, mirroring
+    // `text_refs` / `json_refs`. `args_json_id` is nullable because a
+    // call with no arguments (or one whose args failed to
+    // `JSON.stringify`, e.g. a circular reference) has nothing to store.
+    // `text_id` is nullable too: `text_refs` never stores the empty
+    // string (its dictionary upsert treats `''` as "nothing to dedupe"),
+    // so a call like `console.log()` with zero arguments — whose
+    // `text` beholder reports as `''` — has no `text_refs` row to point
+    // at; `text_id = NULL` there means "empty text", read back as `''`.
+    // `type` keeps its own index for the Console Logs view's type filter.
+    if (!(await instance.schema.hasTable('console_log_items'))) {
+        await instance.raw(`
+			CREATE TABLE console_log_items (
+				id             INTEGER PRIMARY KEY,
+				hash           BLOB NOT NULL UNIQUE,
+				type           TEXT NOT NULL,
+				text_id        INTEGER REFERENCES text_refs(id),
+				args_json_id   INTEGER REFERENCES json_refs(id),
+				loc_url_id     INTEGER REFERENCES url_refs(id),
+				loc_line       INTEGER,
+				loc_column     INTEGER,
+				stack_text_id  INTEGER REFERENCES text_refs(id)
+			)
+		`);
+        await instance.raw('CREATE INDEX idx_console_log_items_type ON console_log_items(type)');
+    }
+    // One row per (page, console log) occurrence — beholder captures a
+    // `ts` per firing, so the same message logged 3 times on one page
+    // yields 3 rows (unlike `anchor_edges`' first-wins dedup: an
+    // occurrence count matters here, not just presence). Replaced
+    // wholesale per page on every non-empty scrape by
+    // `replaceConsoleLogs`, the same Scoped-Replace pattern as
+    // `anchor_edges` / `image_items` — there is no natural key to UPDATE
+    // a specific prior occurrence against.
+    if (!(await instance.schema.hasTable('page_console_logs'))) {
+        await instance.schema.createTable('page_console_logs', (t) => {
+            t.increments('id');
+            t.integer('pageId')
+                .notNullable()
+                .unsigned()
+                .references('content_items.id')
+                .onDelete('CASCADE');
+            t.integer('consoleLogId')
+                .notNullable()
+                .unsigned()
+                .references('console_log_items.id');
+            t.integer('ts').notNullable();
+            t.index('pageId');
+            t.index('consoleLogId');
+        });
+    }
 }

package/lib/archive/create-entity-tables.js

@@ -197,6 +197,7 @@ export async function createEntityTables(instance) {
 			content_length   INTEGER,
 			header_set_id    INTEGER REFERENCES header_sets(id),
 			redirect_dest_id INTEGER REFERENCES content_items(id) DEFERRABLE INITIALLY DEFERRED,
+			alias_of_id      INTEGER REFERENCES content_items(id) DEFERRABLE INITIALLY DEFERRED,
 			source           TEXT NOT NULL DEFAULT 'crawled',
 			first_crawled_at INTEGER,
 			last_crawled_at  INTEGER,
@@ -208,6 +209,15 @@ export async function createEntityTables(instance) {
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_external ON content_items(is_external)');
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_scraped ON content_items(scraped)');
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_redirect_dest_id ON content_items(redirect_dest_id)');
+    // `idx_content_items_alias_of_id` is NOT created here even though
+    // `alias_of_id` is: this DDL runs unconditionally on every archive open
+    // (including legacy archives that still lack the column at this point,
+    // before `migrateContentItemsAliasOfId` adds it), so an unconditional
+    // `CREATE INDEX ... alias_of_id` here would fail with `no such column`
+    // on any archive that predates this feature. The index is created in
+    // `migrateContentItemsAliasOfId` instead, which runs after the
+    // column-add guard for both fresh and legacy archives (same reasoning as
+    // `page_meta.body_hash`'s index).
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_content_type_id ON content_items(content_type_id)');
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_crawl_order ON content_items(crawl_order)');
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_source ON content_items(source)');
@@ -279,10 +289,20 @@ export async function createEntityTables(instance) {
 			main_content_audio_count    INTEGER,
 			main_content_canvas_count   INTEGER,
 			scroll_height_desktop       INTEGER,
-			scroll_height_mobile        INTEGER
+			scroll_height_mobile        INTEGER,
+			body_hash                   BLOB,
+			console_error_count         INTEGER
 		)
 	`);
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_page_meta_og_type ON page_meta(og_type)');
+    // `idx_page_meta_body_hash` is NOT created here even though `body_hash`
+    // is: this DDL runs unconditionally on every archive open (including
+    // legacy archives that still lack the column at this point, before
+    // `migratePageMetaBodyHash` adds it), so an unconditional
+    // `CREATE INDEX ... body_hash` here would fail with `no such column` on
+    // any archive that predates this feature. The index is created in
+    // `migratePageMetaBodyHash` instead, which runs after the column-add
+    // guard for both fresh and legacy archives.
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_page_meta_robots_noindex ON page_meta(robots_noindex)');
     await instance.raw(`
 		CREATE TABLE IF NOT EXISTS resource_items (

package/lib/archive/database.d.ts

@@ -1,6 +1,9 @@
+import type { TemplateClusterReason } from './db-ops/analysis/types.js';
 import type { JsonLdRow, MainContentAudioRow, MainContentButtonRow, MainContentCanvasRow, MainContentHeadingRow, MainContentIframeRow, MainContentImageRow, MainContentTableRow, MainContentVideoRow, TagRow } from './meta/types.js';
-import type { Config, DatabaseOption, DB_Redirect, DB_Resource, DatabaseEvent, InventoryRunMeta, PageFilter, PageSource } from './types.js';
+import type { Config, DatabaseOption, DB_Redirect, DB_Resource, DatabaseEvent, InsertNetworkOutageParams, InventoryRunMeta, PageFilter, PageSource } from './types.js';
+import type { OutageWindow } from '../is-within-outage-window.js';
 import type { PageData, Resource } from '../utils/types/types.js';
+import type { ConsoleLogEntry } from '@d-zero/beholder';
 import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
 import type { Knex } from 'knex';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
@@ -34,6 +37,13 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * database file. Delegates to {@link checkpointOp}.
      */
     checkpoint(): Promise<void>;
+    /**
+     * Closes an outage row by stamping `ended_at` — a no-op if the row is
+     * already closed. Delegates to {@link closeNetworkOutageOp}.
+     * @param id - The `network_outages.id` to close.
+     * @param endedAt - Epoch ms the outage is considered to have ended.
+     */
+    closeNetworkOutage(id: number, endedAt: number): Promise<void>;
     /**
      * Destroys the database connection, releasing all pooled resources.
      * Delegates to {@link destroyOp}.
@@ -277,6 +287,13 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @param urls - URL strings already in `withoutHashAndAuth` form.
      */
     insertInventorySeeds(urls: readonly string[]): Promise<void>;
+    /**
+     * Appends one open (`ended_at = NULL`) row to the `network_outages`
+     * journal. Delegates to {@link insertNetworkOutageOp}.
+     * @param params - The confirmed-outage fields to record.
+     * @returns The autoincremented `id` of the newly-inserted row.
+     */
+    insertNetworkOutage(params: InsertNetworkOutageParams): Promise<number>;
     /**
      * Records a partial scrape failure against the page identified by `url`.
      * Delegates to {@link insertPageErrorOp}.
@@ -307,6 +324,13 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @returns Lower-cased hostnames safe to short-circuit.
      */
     listDnsBurnedHostCandidates(): Promise<string[]>;
+    /**
+     * Lists every recorded outage as a resolved {@link OutageWindow}.
+     * Delegates to {@link listNetworkOutagesOp}.
+     * @returns Resolved outage windows, or `[]` on an archive that predates
+     *   `network_outages` or has recorded no outages.
+     */
+    listNetworkOutages(): Promise<OutageWindow[]>;
     /**
      * Appends one row to the `inventory_runs` audit log.
      * Delegates to {@link recordInventoryRunOp}.
@@ -339,12 +363,23 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
         line?: number | null;
         col?: number | null;
     }[]): Promise<void>;
+    /**
+     * Replaces one page's `page_console_logs` rows with a freshly captured
+     * set of console messages / page errors. Delegates to
+     * {@link replaceConsoleLogsOp}.
+     * @param pageUrl - The originally-requested URL, normalised (`withoutHashAndAuth` form).
+     * @param redirectPaths - The redirect chain hops captured during fetch, in order.
+     * @param entries - The console log entries to persist.
+     */
+    replaceConsoleLogs(pageUrl: string, redirectPaths: readonly string[], entries: readonly ConsoleLogEntry[]): Promise<void>;
     /**
      * Replaces the stored DOM-structure template classification with a
      * freshly generated set. Delegates to {@link replacePageTemplatesOp}.
      * @param templateKeysByUrl - Page URL → template key.
+     * @param clusterReasonsByTemplateKey - Template key → cluster-selection
+     *   evidence, if the caller captured it.
      */
-    replacePageTemplates(templateKeysByUrl: ReadonlyMap<string, string>): Promise<void>;
+    replacePageTemplates(templateKeysByUrl: ReadonlyMap<string, string>, clusterReasonsByTemplateKey?: ReadonlyMap<string, TemplateClusterReason>): Promise<void>;
     /**
      * Promote previously-external pages whose URL falls under any of the new
      * scope entries back to a "needs scraping" state.