@nitpicker/crawler 0.13.0 → 0.15.0

package/lib/archive/create-adjunct-tables.js

@@ -8,11 +8,29 @@
  *   an external link that failed DNS, or null for a process-level error)
  * - `page_tags` — Wappalyzer detections, FK → `content_items(id)`
  * - `page_jsonld` — JSON-LD / SpeculationRules, FK → `content_items(id)`
+ * - `page_main_content_headings` / `_images` / `_tables` / `_buttons` /
+ *   `_iframes` / `_videos` / `_audios` / `_canvases` — beholder
+ *   `MainContentsData` sub-entity arrays, one row per DOM element, FK →
+ *   `content_items(id)`
  * - `inventory_runs` — `--inventory` audit log (no FK; append-only)
+ * - `network_outages` — operator-network-outage journal (no FK; append-only
+ *   except `ended_at`, which is written once on recovery)
  * - `analysis_text_refs` + `analysis_violations` — analyze-phase findings,
  *   FK → `content_items(id)`
+ * - `page_templates` — DOM-structure template classification (`--templates`,
+ *   `@nitpicker/core`'s `template-classification/`), one row per classified
+ *   page, FK → `content_items(id)`
+ * - `page_template_clusters` — one row per distinct `page_templates.template_key`,
+ *   holding `@d-zero/page-cluster`'s cluster-selection evidence (no FK;
+ *   `template_key` is not a `page_templates` FK target, so consistency is
+ *   maintained by replacing both tables together, not by a foreign key)
  * - `page_html_blobs` + `page_html_ref` — content-addressable HTML
  *   snapshots, FK → `content_items(id)`
+ * - `console_log_items` — content-addressable dictionary of distinct
+ *   console messages / page errors (no FK; hash-deduplicated across every
+ *   page in the archive, mirroring `text_refs` / `json_refs`)
+ * - `page_console_logs` — one row per (page, console log) occurrence, FK →
+ *   `content_items(id)` and `console_log_items(id)`
  *
  * The DDL is shared between fresh-archive provisioning ({@link initSchema}
  * calls this right after `createEntityTables`) and the migration script
@@ -129,6 +147,134 @@ export async function createAdjunctTables(instance) {
         // `list_pages_by_jsonld_type` JOINs.
         await instance.raw('CREATE INDEX page_jsonld_type_pageId ON page_jsonld(type, pageId)');
     }
+    // Beholder `MainContentsData` sub-entities, one adjunct table per array
+    // (headings/images/tables/buttons/iframes/videos/audios/canvases). Same
+    // shape as `page_tags` / `page_jsonld`: `pageId` FK → `content_items(id)`
+    // ON DELETE CASCADE, individually guarded so any subset can pre-exist.
+    // `order` preserves the DOM traversal order beholder returns the array
+    // in (0-based); it is not itself an index target since these tables are
+    // always read whole-page via `WHERE pageId = ? ORDER BY "order"`.
+    if (!(await instance.schema.hasTable('page_main_content_headings'))) {
+        await instance.schema.createTable('page_main_content_headings', (t) => {
+            t.increments('id');
+            t.integer('pageId')
+                .notNullable()
+                .unsigned()
+                .references('content_items.id')
+                .onDelete('CASCADE');
+            t.integer('order').notNullable();
+            t.text('text');
+            t.integer('level').notNullable();
+            t.index('pageId');
+        });
+    }
+    if (!(await instance.schema.hasTable('page_main_content_images'))) {
+        await instance.schema.createTable('page_main_content_images', (t) => {
+            t.increments('id');
+            t.integer('pageId')
+                .notNullable()
+                .unsigned()
+                .references('content_items.id')
+                .onDelete('CASCADE');
+            t.integer('order').notNullable();
+            t.string('src', 8190).notNullable();
+            t.text('alt').notNullable();
+            t.index('pageId');
+        });
+    }
+    if (!(await instance.schema.hasTable('page_main_content_tables'))) {
+        await instance.schema.createTable('page_main_content_tables', (t) => {
+            t.increments('id');
+            t.integer('pageId')
+                .notNullable()
+                .unsigned()
+                .references('content_items.id')
+                .onDelete('CASCADE');
+            t.integer('order').notNullable();
+            t.integer('rows').notNullable();
+            t.integer('cols').notNullable();
+            t.boolean('hasHeader').notNullable();
+            t.boolean('hasFooter').notNullable();
+            t.boolean('hasMergedCell').notNullable();
+            t.index('pageId');
+        });
+    }
+    if (!(await instance.schema.hasTable('page_main_content_buttons'))) {
+        await instance.schema.createTable('page_main_content_buttons', (t) => {
+            t.increments('id');
+            t.integer('pageId')
+                .notNullable()
+                .unsigned()
+                .references('content_items.id')
+                .onDelete('CASCADE');
+            t.integer('order').notNullable();
+            t.string('nodeName').notNullable();
+            t.string('role');
+            t.string('type');
+            t.text('text');
+            t.boolean('disabled').notNullable();
+            t.index('pageId');
+        });
+    }
+    if (!(await instance.schema.hasTable('page_main_content_iframes'))) {
+        await instance.schema.createTable('page_main_content_iframes', (t) => {
+            t.increments('id');
+            t.integer('pageId')
+                .notNullable()
+                .unsigned()
+                .references('content_items.id')
+                .onDelete('CASCADE');
+            t.integer('order').notNullable();
+            t.string('src', 8190).notNullable();
+            t.text('title');
+            t.string('width');
+            t.string('height');
+            t.index('pageId');
+        });
+    }
+    if (!(await instance.schema.hasTable('page_main_content_videos'))) {
+        await instance.schema.createTable('page_main_content_videos', (t) => {
+            t.increments('id');
+            t.integer('pageId')
+                .notNullable()
+                .unsigned()
+                .references('content_items.id')
+                .onDelete('CASCADE');
+            t.integer('order').notNullable();
+            t.string('src', 8190).notNullable();
+            t.string('poster', 8190);
+            t.integer('width').notNullable();
+            t.integer('height').notNullable();
+            t.index('pageId');
+        });
+    }
+    if (!(await instance.schema.hasTable('page_main_content_audios'))) {
+        await instance.schema.createTable('page_main_content_audios', (t) => {
+            t.increments('id');
+            t.integer('pageId')
+                .notNullable()
+                .unsigned()
+                .references('content_items.id')
+                .onDelete('CASCADE');
+            t.integer('order').notNullable();
+            t.string('src', 8190).notNullable();
+            t.index('pageId');
+        });
+    }
+    if (!(await instance.schema.hasTable('page_main_content_canvases'))) {
+        await instance.schema.createTable('page_main_content_canvases', (t) => {
+            t.increments('id');
+            t.integer('pageId')
+                .notNullable()
+                .unsigned()
+                .references('content_items.id')
+                .onDelete('CASCADE');
+            t.integer('order').notNullable();
+            t.integer('width').notNullable();
+            t.integer('height').notNullable();
+            t.index('pageId');
+        });
+    }
     if (!(await instance.schema.hasTable('inventory_runs'))) {
         await instance.schema.createTable('inventory_runs', (t) => {
             // One row per successful `--inventory <list>` invocation. The
@@ -144,10 +290,42 @@ export async function createAdjunctTables(instance) {
             t.integer('new_pages').nullable();
             t.integer('new_resources').nullable();
             t.integer('scope_skipped').nullable();
+            t.integer('invalid_skipped').nullable();
             t.text('notes').nullable();
             t.index('ran_at');
         });
     }
+    if (!(await instance.schema.hasTable('network_outages'))) {
+        await instance.schema.createTable('network_outages', (t) => {
+            // One row per detected operator-network outage. The archive's
+            // evidence that a run of `dns`/`local-network`-shaped failures
+            // was caused by the crawl operator's own connectivity, not the
+            // target sites — used to retroactively re-classify `crawl_errors`
+            // / `page_errors` rows whose `createdAt` falls inside a window
+            // (see `is-within-outage-window.ts`). No index: a crawl session
+            // produces at most a handful of these rows, so a full in-memory
+            // scan by every consumer is cheaper than maintaining a B-tree
+            // that few queries would use (`ARCHITECTURE.md`'s
+            // perf-index-is-not-free / evidence-before-indexing rules).
+            t.increments('id');
+            // Backdated to the earliest error still inside the detector's
+            // sliding window at trigger time, NOT the trigger instant —
+            // see `NetworkOutageDetector`'s `OutageSuspect.startedAt`.
+            t.integer('started_at').notNullable();
+            // When the sliding window actually crossed both thresholds.
+            t.integer('detected_at').notNullable();
+            // NULL until a recovery probe succeeds. A row can also be left
+            // NULL forever if the crawl process is killed mid-outage; readers
+            // MUST NOT treat a NULL `ended_at` as an unbounded window (that
+            // would retroactively cover every later error as
+            // network-caused) — see `is-within-outage-window.ts` and the
+            // writer-side clamp-on-next-open in `db-ops/outages/`.
+            t.integer('ended_at').nullable();
+            t.string('probe_host').nullable();
+            t.integer('trigger_error_count').notNullable();
+            t.integer('trigger_host_count').notNullable();
+        });
+    }
     if (!(await instance.schema.hasTable('analysis_text_refs'))) {
         await instance.raw(`
 			CREATE TABLE analysis_text_refs (
@@ -184,6 +362,49 @@ export async function createAdjunctTables(instance) {
         await instance.raw('CREATE INDEX av_code_order ON analysis_violations(code_sort_key, id)');
         await instance.raw('CREATE INDEX av_page ON analysis_violations(page_id, id)');
     }
+    // DOM-structure template classification (`--templates`). One row per
+    // internal HTML page that was classified; `page_id` is both the PK and
+    // the natural key (1:1 with `content_items`), so — unlike
+    // `analysis_violations`, which is 1:many and needs a surrogate `id` —
+    // there's nothing to index beyond the PK itself. `WITHOUT ROWID` packs
+    // rows directly in the PK b-tree, matching `page_html_ref`'s shape
+    // (small fixed-width row, PK-only lookups).
+    if (!(await instance.schema.hasTable('page_templates'))) {
+        await instance.raw(`
+			CREATE TABLE page_templates (
+				page_id      INTEGER PRIMARY KEY REFERENCES content_items(id),
+				template_key TEXT NOT NULL
+			) WITHOUT ROWID
+		`);
+    }
+    // One row per distinct `template_key` produced by the same `--templates`
+    // classification run, holding `@d-zero/page-cluster`'s cluster-selection
+    // evidence (`ClusterReason`, renamed `TemplateClusterReason` on this side)
+    // as a zstd-compressed JSON blob — same BLOB+codec+size shape as
+    // `page_html_blobs` below. A column on `page_templates` was rejected: that
+    // table is one row per *page*, so the same cluster's reason would be
+    // duplicated across every member page (multi-GB on a large archive with a
+    // few-hundred-member cluster). A `json_refs` row was also rejected: reason
+    // payloads differ per cluster (distinct `memberCount`/token sets), so
+    // content-address dedup would not pay for itself, and `json_refs` is a
+    // shared dictionary that other tables reference — this table's full
+    // replace-on-every-run write pattern (see `replacePageTemplates`) would
+    // otherwise leave orphaned rows behind with no owner able to delete them.
+    // No FK to `page_templates`: `template_key` is not that table's primary
+    // key (`page_id` is), so there is nothing to reference — consistency is
+    // instead maintained by replacing both tables in the same transaction.
+    if (!(await instance.schema.hasTable('page_template_clusters'))) {
+        await instance.raw(`
+			CREATE TABLE page_template_clusters (
+				template_key TEXT PRIMARY KEY,
+				member_count INTEGER NOT NULL,
+				reason_json  BLOB NOT NULL,
+				codec        TEXT NOT NULL CHECK(codec IN ('zstd', 'none')),
+				size_raw     INTEGER NOT NULL,
+				size_stored  INTEGER NOT NULL
+			) WITHOUT ROWID
+		`);
+    }
     // Content-addressable HTML blob storage. Knex's schema builder doesn't
     // expose a WITHOUT ROWID toggle, so the BLOB tables are created via raw
     // SQL. WITHOUT ROWID keeps the rows packed inside the b-tree leaves
@@ -210,4 +431,60 @@ export async function createAdjunctTables(instance) {
 		`);
         await instance.raw('CREATE INDEX idx_page_html_ref_hash ON page_html_ref(hash)');
     }
+    // Content-addressable dictionary of distinct console messages / page
+    // errors (beholder's `ConsoleLogEntry`, issue #228). `hash` is
+    // SHA-256 over a canonical tuple of every content field (type, text,
+    // args JSON, location, stack) — the same identical warning emitted by
+    // a shared framework on every page therefore collapses to one row
+    // regardless of how many pages or how many times it fires, mirroring
+    // `text_refs` / `json_refs`. `args_json_id` is nullable because a
+    // call with no arguments (or one whose args failed to
+    // `JSON.stringify`, e.g. a circular reference) has nothing to store.
+    // `text_id` is nullable too: `text_refs` never stores the empty
+    // string (its dictionary upsert treats `''` as "nothing to dedupe"),
+    // so a call like `console.log()` with zero arguments — whose
+    // `text` beholder reports as `''` — has no `text_refs` row to point
+    // at; `text_id = NULL` there means "empty text", read back as `''`.
+    // `type` keeps its own index for the Console Logs view's type filter.
+    if (!(await instance.schema.hasTable('console_log_items'))) {
+        await instance.raw(`
+			CREATE TABLE console_log_items (
+				id             INTEGER PRIMARY KEY,
+				hash           BLOB NOT NULL UNIQUE,
+				type           TEXT NOT NULL,
+				text_id        INTEGER REFERENCES text_refs(id),
+				args_json_id   INTEGER REFERENCES json_refs(id),
+				loc_url_id     INTEGER REFERENCES url_refs(id),
+				loc_line       INTEGER,
+				loc_column     INTEGER,
+				stack_text_id  INTEGER REFERENCES text_refs(id)
+			)
+		`);
+        await instance.raw('CREATE INDEX idx_console_log_items_type ON console_log_items(type)');
+    }
+    // One row per (page, console log) occurrence — beholder captures a
+    // `ts` per firing, so the same message logged 3 times on one page
+    // yields 3 rows (unlike `anchor_edges`' first-wins dedup: an
+    // occurrence count matters here, not just presence). Replaced
+    // wholesale per page on every non-empty scrape by
+    // `replaceConsoleLogs`, the same Scoped-Replace pattern as
+    // `anchor_edges` / `image_items` — there is no natural key to UPDATE
+    // a specific prior occurrence against.
+    if (!(await instance.schema.hasTable('page_console_logs'))) {
+        await instance.schema.createTable('page_console_logs', (t) => {
+            t.increments('id');
+            t.integer('pageId')
+                .notNullable()
+                .unsigned()
+                .references('content_items.id')
+                .onDelete('CASCADE');
+            t.integer('consoleLogId')
+                .notNullable()
+                .unsigned()
+                .references('console_log_items.id');
+            t.integer('ts').notNullable();
+            t.index('pageId');
+            t.index('consoleLogId');
+        });
+    }
 }

package/lib/archive/create-entity-tables.d.ts

@@ -136,6 +136,20 @@ import type { Knex } from 'knex';
  * routing so the two entities never disagree on which values count as
  * "large data URI".
  *
+ * **`page_meta.main_content_*` / `scroll_height_*` columns.** Denormalised
+ * aggregates derived from beholder's `MainContentsData` / `ScrollHeightData`
+ * (word/element counts, desktop+mobile scroll height), following the same
+ * write-once-at-scrape-time pattern as `tag_count` / `jsonld_count` so list
+ * / detail reads never re-derive them from the per-page child tables
+ * (`page_main_content_headings` etc., see `create-adjunct-tables.ts`).
+ * `main_content_node_name` / `_id` / `_role` / `_selector` / `_class_list`
+ * identify the detected main-content element; unlike `title_text_id` /
+ * `description_text_id` these are stored as plain `TEXT` rather than routed
+ * through `text_refs` — the values are page-specific diagnostics with low
+ * cross-page reuse, so the ref-table dedup machinery would add write-path
+ * cost without a corresponding storage win. `main_content_class_list` holds
+ * a JSON-encoded string array.
+ *
  * ### Index rationale
  *
  * Every index below reflects a legacy-baseline single-column index that

package/lib/archive/create-entity-tables.js

@@ -135,6 +135,20 @@
  * routing so the two entities never disagree on which values count as
  * "large data URI".
  *
+ * **`page_meta.main_content_*` / `scroll_height_*` columns.** Denormalised
+ * aggregates derived from beholder's `MainContentsData` / `ScrollHeightData`
+ * (word/element counts, desktop+mobile scroll height), following the same
+ * write-once-at-scrape-time pattern as `tag_count` / `jsonld_count` so list
+ * / detail reads never re-derive them from the per-page child tables
+ * (`page_main_content_headings` etc., see `create-adjunct-tables.ts`).
+ * `main_content_node_name` / `_id` / `_role` / `_selector` / `_class_list`
+ * identify the detected main-content element; unlike `title_text_id` /
+ * `description_text_id` these are stored as plain `TEXT` rather than routed
+ * through `text_refs` — the values are page-specific diagnostics with low
+ * cross-page reuse, so the ref-table dedup machinery would add write-path
+ * cost without a corresponding storage win. `main_content_class_list` holds
+ * a JSON-encoded string array.
+ *
  * ### Index rationale
  *
  * Every index below reflects a legacy-baseline single-column index that
@@ -183,6 +197,7 @@ export async function createEntityTables(instance) {
 			content_length   INTEGER,
 			header_set_id    INTEGER REFERENCES header_sets(id),
 			redirect_dest_id INTEGER REFERENCES content_items(id) DEFERRABLE INITIALLY DEFERRED,
+			alias_of_id      INTEGER REFERENCES content_items(id) DEFERRABLE INITIALLY DEFERRED,
 			source           TEXT NOT NULL DEFAULT 'crawled',
 			first_crawled_at INTEGER,
 			last_crawled_at  INTEGER,
@@ -194,6 +209,15 @@ export async function createEntityTables(instance) {
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_external ON content_items(is_external)');
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_scraped ON content_items(scraped)');
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_redirect_dest_id ON content_items(redirect_dest_id)');
+    // `idx_content_items_alias_of_id` is NOT created here even though
+    // `alias_of_id` is: this DDL runs unconditionally on every archive open
+    // (including legacy archives that still lack the column at this point,
+    // before `migrateContentItemsAliasOfId` adds it), so an unconditional
+    // `CREATE INDEX ... alias_of_id` here would fail with `no such column`
+    // on any archive that predates this feature. The index is created in
+    // `migrateContentItemsAliasOfId` instead, which runs after the
+    // column-add guard for both fresh and legacy archives (same reasoning as
+    // `page_meta.body_hash`'s index).
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_content_type_id ON content_items(content_type_id)');
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_crawl_order ON content_items(crawl_order)');
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_source ON content_items(source)');
@@ -248,10 +272,37 @@ export async function createEntityTables(instance) {
 			tag_count                   INTEGER,
 			jsonld_count                INTEGER,
 			tags_providers_csv          TEXT,
-			meta_extras_json_id         INTEGER REFERENCES json_refs(id)
+			meta_extras_json_id         INTEGER REFERENCES json_refs(id),
+			main_content_node_name      TEXT,
+			main_content_id             TEXT,
+			main_content_role           TEXT,
+			main_content_selector       TEXT,
+			main_content_class_list     TEXT,
+			main_content_word_count     INTEGER,
+			main_content_body_word_count INTEGER,
+			main_content_heading_count  INTEGER,
+			main_content_image_count    INTEGER,
+			main_content_table_count    INTEGER,
+			main_content_button_count   INTEGER,
+			main_content_iframe_count   INTEGER,
+			main_content_video_count    INTEGER,
+			main_content_audio_count    INTEGER,
+			main_content_canvas_count   INTEGER,
+			scroll_height_desktop       INTEGER,
+			scroll_height_mobile        INTEGER,
+			body_hash                   BLOB,
+			console_error_count         INTEGER
 		)
 	`);
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_page_meta_og_type ON page_meta(og_type)');
+    // `idx_page_meta_body_hash` is NOT created here even though `body_hash`
+    // is: this DDL runs unconditionally on every archive open (including
+    // legacy archives that still lack the column at this point, before
+    // `migratePageMetaBodyHash` adds it), so an unconditional
+    // `CREATE INDEX ... body_hash` here would fail with `no such column` on
+    // any archive that predates this feature. The index is created in
+    // `migratePageMetaBodyHash` instead, which runs after the column-add
+    // guard for both fresh and legacy archives.
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_page_meta_robots_noindex ON page_meta(robots_noindex)');
     await instance.raw(`
 		CREATE TABLE IF NOT EXISTS resource_items (

package/lib/archive/database.d.ts

@@ -1,6 +1,9 @@
-import type { JsonLdRow, TagRow } from './meta/types.js';
-import type { Config, DatabaseOption, DB_Redirect, DB_Resource, DatabaseEvent, InventoryRunMeta, PageFilter, PageSource } from './types.js';
+import type { TemplateClusterReason } from './db-ops/analysis/types.js';
+import type { JsonLdRow, MainContentAudioRow, MainContentButtonRow, MainContentCanvasRow, MainContentHeadingRow, MainContentIframeRow, MainContentImageRow, MainContentTableRow, MainContentVideoRow, TagRow } from './meta/types.js';
+import type { Config, DatabaseOption, DB_Redirect, DB_Resource, DatabaseEvent, InsertNetworkOutageParams, InventoryRunMeta, PageFilter, PageSource } from './types.js';
+import type { OutageWindow } from '../is-within-outage-window.js';
 import type { PageData, Resource } from '../utils/types/types.js';
+import type { ConsoleLogEntry } from '@d-zero/beholder';
 import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
 import type { Knex } from 'knex';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
@@ -34,6 +37,13 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * database file. Delegates to {@link checkpointOp}.
      */
     checkpoint(): Promise<void>;
+    /**
+     * Closes an outage row by stamping `ended_at` — a no-op if the row is
+     * already closed. Delegates to {@link closeNetworkOutageOp}.
+     * @param id - The `network_outages.id` to close.
+     * @param endedAt - Epoch ms the outage is considered to have ended.
+     */
+    closeNetworkOutage(id: number, endedAt: number): Promise<void>;
     /**
      * Destroys the database connection, releasing all pooled resources.
      * Delegates to {@link destroyOp}.
@@ -46,6 +56,12 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @returns An array of anchor records with resolved URL, title, status, and content type.
      */
     getAnchorsOnPage(pageId: number): Promise<any[]>;
+    /**
+     * Retrieves all `page_main_content_audios` rows for the given page id.
+     * Delegates to {@link getAudiosOfPageOp}.
+     * @param pageId
+     */
+    getAudiosOfPage(pageId: number): Promise<MainContentAudioRow[]>;
     /**
      * Retrieves the base URL of the crawl session from the `info` table.
      * Delegates to {@link getBaseUrlOp}.
@@ -53,6 +69,18 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @throws {Error} If no base URL is found in the database.
      */
     getBaseUrl(): Promise<string>;
+    /**
+     * Retrieves all `page_main_content_buttons` rows for the given page id.
+     * Delegates to {@link getButtonsOfPageOp}.
+     * @param pageId
+     */
+    getButtonsOfPage(pageId: number): Promise<MainContentButtonRow[]>;
+    /**
+     * Retrieves all `page_main_content_canvases` rows for the given page id.
+     * Delegates to {@link getCanvasesOfPageOp}.
+     * @param pageId
+     */
+    getCanvasesOfPage(pageId: number): Promise<MainContentCanvasRow[]>;
     /**
      * Retrieves the full crawl configuration from the `info` table.
      * Delegates to {@link getConfigOp}.
@@ -85,6 +113,12 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @returns URLs found in `resources`.
      */
     getExistingResourceUrls(urls: readonly string[]): Promise<string[]>;
+    /**
+     * Retrieves all `page_main_content_headings` rows for the given page id.
+     * Delegates to {@link getHeadingsOfPageOp}.
+     * @param pageId
+     */
+    getHeadingsOfPage(pageId: number): Promise<MainContentHeadingRow[]>;
     /**
      * Reads the HTML snapshot stored as a zstd-compressed BLOB for the given page.
      * Delegates to {@link getHtmlOfPageByIdOp}.
@@ -92,6 +126,12 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @returns The decompressed HTML string, or `null` if no snapshot is stored.
      */
     getHtmlOfPageById(pageId: number): Promise<string | null>;
+    /**
+     * Retrieves all `page_main_content_iframes` rows for the given page id.
+     * Delegates to {@link getIframesOfPageOp}.
+     * @param pageId
+     */
+    getIframesOfPage(pageId: number): Promise<MainContentIframeRow[]>;
     /**
      * Retrieves all `page_jsonld` rows for the given page id, parsed back into
      * {@link JsonLdRow} shape. Delegates to {@link getJsonLdOfPageOp}.
@@ -105,6 +145,18 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @returns The Knex instance connected to the SQLite database.
      */
     getKnex(): Knex;
+    /**
+     * Retrieves all `page_main_content_images` rows for the given page id.
+     * Delegates to {@link getMainContentImagesOfPageOp}.
+     * @param pageId
+     */
+    getMainContentImagesOfPage(pageId: number): Promise<MainContentImageRow[]>;
+    /**
+     * Retrieves all `page_main_content_tables` rows for the given page id.
+     * Delegates to {@link getMainContentTablesOfPageOp}.
+     * @param pageId
+     */
+    getMainContentTablesOfPage(pageId: number): Promise<MainContentTableRow[]>;
     /**
      * Retrieves the crawl session name from the `info` table.
      * Delegates to {@link getNameOp}.
@@ -207,6 +259,12 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @param pageId
      */
     getTagsOfPage(pageId: number): Promise<TagRow[]>;
+    /**
+     * Retrieves all `page_main_content_videos` rows for the given page id.
+     * Delegates to {@link getVideosOfPageOp}.
+     * @param pageId
+     */
+    getVideosOfPage(pageId: number): Promise<MainContentVideoRow[]>;
     /**
      * Records a crawler-level (`error` channel) failure into `crawl_errors`.
      * Delegates to {@link insertCrawlErrorOp}.
@@ -229,6 +287,13 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @param urls - URL strings already in `withoutHashAndAuth` form.
      */
     insertInventorySeeds(urls: readonly string[]): Promise<void>;
+    /**
+     * Appends one open (`ended_at = NULL`) row to the `network_outages`
+     * journal. Delegates to {@link insertNetworkOutageOp}.
+     * @param params - The confirmed-outage fields to record.
+     * @returns The autoincremented `id` of the newly-inserted row.
+     */
+    insertNetworkOutage(params: InsertNetworkOutageParams): Promise<number>;
     /**
      * Records a partial scrape failure against the page identified by `url`.
      * Delegates to {@link insertPageErrorOp}.
@@ -259,6 +324,13 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @returns Lower-cased hostnames safe to short-circuit.
      */
     listDnsBurnedHostCandidates(): Promise<string[]>;
+    /**
+     * Lists every recorded outage as a resolved {@link OutageWindow}.
+     * Delegates to {@link listNetworkOutagesOp}.
+     * @returns Resolved outage windows, or `[]` on an archive that predates
+     *   `network_outages` or has recorded no outages.
+     */
+    listNetworkOutages(): Promise<OutageWindow[]>;
     /**
      * Appends one row to the `inventory_runs` audit log.
      * Delegates to {@link recordInventoryRunOp}.
@@ -291,6 +363,23 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
         line?: number | null;
         col?: number | null;
     }[]): Promise<void>;
+    /**
+     * Replaces one page's `page_console_logs` rows with a freshly captured
+     * set of console messages / page errors. Delegates to
+     * {@link replaceConsoleLogsOp}.
+     * @param pageUrl - The originally-requested URL, normalised (`withoutHashAndAuth` form).
+     * @param redirectPaths - The redirect chain hops captured during fetch, in order.
+     * @param entries - The console log entries to persist.
+     */
+    replaceConsoleLogs(pageUrl: string, redirectPaths: readonly string[], entries: readonly ConsoleLogEntry[]): Promise<void>;
+    /**
+     * Replaces the stored DOM-structure template classification with a
+     * freshly generated set. Delegates to {@link replacePageTemplatesOp}.
+     * @param templateKeysByUrl - Page URL → template key.
+     * @param clusterReasonsByTemplateKey - Template key → cluster-selection
+     *   evidence, if the caller captured it.
+     */
+    replacePageTemplates(templateKeysByUrl: ReadonlyMap<string, string>, clusterReasonsByTemplateKey?: ReadonlyMap<string, TemplateClusterReason>): Promise<void>;
     /**
      * Promote previously-external pages whose URL falls under any of the new
      * scope entries back to a "needs scraping" state.