@nitpicker/crawler 0.13.0 → 0.14.0

package/lib/archive/archive-accessor.d.ts

@@ -68,6 +68,27 @@ export declare class ArchiveAccessor extends EventEmitter<DatabaseEvent> {
      * @returns An array of anchor records found on the page.
      */
     getAnchorsOnPage(pageId: number): Promise<any[]>;
+    /**
+     * Retrieves the audios within the given page's detected main content
+     * region, from `page_main_content_audios`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered audio rows.
+     */
+    getAudiosOfPage(pageId: number): Promise<import("./meta/types.js").MainContentAudioRow[]>;
+    /**
+     * Retrieves the button-like elements within the given page's detected
+     * main content region, from `page_main_content_buttons`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered button rows.
+     */
+    getButtonsOfPage(pageId: number): Promise<import("./meta/types.js").MainContentButtonRow[]>;
+    /**
+     * Retrieves the canvases within the given page's detected main content
+     * region, from `page_main_content_canvases`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered canvas rows.
+     */
+    getCanvasesOfPage(pageId: number): Promise<import("./meta/types.js").MainContentCanvasRow[]>;
     /**
      * Retrieves the crawl configuration stored in the archive database.
      * @returns The parsed {@link Config} object.
@@ -87,6 +108,13 @@ export declare class ArchiveAccessor extends EventEmitter<DatabaseEvent> {
      * @returns The file contents as a string.
      */
     getData(name: string, format?: 'txt' | 'html'): Promise<string>;
+    /**
+     * Retrieves the headings within the given page's detected main content
+     * region, from `page_main_content_headings`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered heading rows.
+     */
+    getHeadingsOfPage(pageId: number): Promise<import("./meta/types.js").MainContentHeadingRow[]>;
     /**
      * Reads the HTML snapshot of a page from the archive.
      *
@@ -117,6 +145,13 @@ export declare class ArchiveAccessor extends EventEmitter<DatabaseEvent> {
      * }
      */
     getHtmlOfPage(pageId: number): Promise<string | null>;
+    /**
+     * Retrieves the iframes within the given page's detected main content
+     * region, from `page_main_content_iframes`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered iframe rows.
+     */
+    getIframesOfPage(pageId: number): Promise<import("./meta/types.js").MainContentIframeRow[]>;
     /**
      * Retrieves the JSON-LD / SpeculationRules entries for the given page,
      * parsed back from the `page_jsonld` table.
@@ -131,6 +166,20 @@ export declare class ArchiveAccessor extends EventEmitter<DatabaseEvent> {
      * @returns The Knex instance connected to the SQLite database.
      */
     getKnex(): import("knex").Knex<any, any[]>;
+    /**
+     * Retrieves the images within the given page's detected main content
+     * region, from `page_main_content_images`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered image rows.
+     */
+    getMainContentImagesOfPage(pageId: number): Promise<import("./meta/types.js").MainContentImageRow[]>;
+    /**
+     * Retrieves the tables within the given page's detected main content
+     * region, from `page_main_content_tables`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered table rows.
+     */
+    getMainContentTablesOfPage(pageId: number): Promise<import("./meta/types.js").MainContentTableRow[]>;
     /**
      * Retrieves all pages from the archive, optionally filtered by type.
      * Eagerly loads redirect relationships (`redirectFrom`) but does NOT load
@@ -179,6 +228,13 @@ export declare class ArchiveAccessor extends EventEmitter<DatabaseEvent> {
      * @returns Ordered entries with provider, category, externalId, etc.
      */
     getTagsOfPage(pageId: number): Promise<import("./meta/types.js").TagRow[]>;
+    /**
+     * Retrieves the videos within the given page's detected main content
+     * region, from `page_main_content_videos`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered video rows.
+     */
+    getVideosOfPage(pageId: number): Promise<import("./meta/types.js").MainContentVideoRow[]>;
     /**
      * Stores custom data in the archive under the configured namespace.
      * Requires a namespace to be set on this accessor; throws if namespace is null.

package/lib/archive/archive-accessor.js

@@ -128,6 +128,33 @@ export class ArchiveAccessor extends EventEmitter {
         const refs = await this.#db.getAnchorsOnPage(pageId);
         return refs;
     }
+    /**
+     * Retrieves the audios within the given page's detected main content
+     * region, from `page_main_content_audios`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered audio rows.
+     */
+    async getAudiosOfPage(pageId) {
+        return this.#db.getAudiosOfPage(pageId);
+    }
+    /**
+     * Retrieves the button-like elements within the given page's detected
+     * main content region, from `page_main_content_buttons`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered button rows.
+     */
+    async getButtonsOfPage(pageId) {
+        return this.#db.getButtonsOfPage(pageId);
+    }
+    /**
+     * Retrieves the canvases within the given page's detected main content
+     * region, from `page_main_content_canvases`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered canvas rows.
+     */
+    async getCanvasesOfPage(pageId) {
+        return this.#db.getCanvasesOfPage(pageId);
+    }
     /**
      * Retrieves the crawl configuration stored in the archive database.
      * @returns The parsed {@link Config} object.
@@ -143,6 +170,15 @@ export class ArchiveAccessor extends EventEmitter {
         }
         return await readText(filePath);
     }
+    /**
+     * Retrieves the headings within the given page's detected main content
+     * region, from `page_main_content_headings`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered heading rows.
+     */
+    async getHeadingsOfPage(pageId) {
+        return this.#db.getHeadingsOfPage(pageId);
+    }
     /**
      * Reads the HTML snapshot of a page from the archive.
      *
@@ -179,6 +215,15 @@ export class ArchiveAccessor extends EventEmitter {
         }
         return html;
     }
+    /**
+     * Retrieves the iframes within the given page's detected main content
+     * region, from `page_main_content_iframes`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered iframe rows.
+     */
+    async getIframesOfPage(pageId) {
+        return this.#db.getIframesOfPage(pageId);
+    }
     /**
      * Retrieves the JSON-LD / SpeculationRules entries for the given page,
      * parsed back from the `page_jsonld` table.
@@ -197,6 +242,24 @@ export class ArchiveAccessor extends EventEmitter {
     getKnex() {
         return this.#db.getKnex();
     }
+    /**
+     * Retrieves the images within the given page's detected main content
+     * region, from `page_main_content_images`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered image rows.
+     */
+    async getMainContentImagesOfPage(pageId) {
+        return this.#db.getMainContentImagesOfPage(pageId);
+    }
+    /**
+     * Retrieves the tables within the given page's detected main content
+     * region, from `page_main_content_tables`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered table rows.
+     */
+    async getMainContentTablesOfPage(pageId) {
+        return this.#db.getMainContentTablesOfPage(pageId);
+    }
     /**
      * Retrieves all pages from the archive, optionally filtered by type.
      * Eagerly loads redirect relationships (`redirectFrom`) but does NOT load
@@ -289,6 +352,15 @@ export class ArchiveAccessor extends EventEmitter {
     async getTagsOfPage(pageId) {
         return this.#db.getTagsOfPage(pageId);
     }
+    /**
+     * Retrieves the videos within the given page's detected main content
+     * region, from `page_main_content_videos`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered video rows.
+     */
+    async getVideosOfPage(pageId) {
+        return this.#db.getVideosOfPage(pageId);
+    }
     /**
      * Stores custom data in the archive under the configured namespace.
      * Requires a namespace to be set on this accessor; throws if namespace is null.

package/lib/archive/archive.d.ts

@@ -206,6 +206,17 @@ export default class Archive extends ArchiveAccessor {
         line?: number | null;
         col?: number | null;
     }[]): Promise<void>;
+    /**
+     * Replaces the archive's DOM-structure template classification
+     * (`--templates`) with a fresh SQL-backed set.
+     *
+     * Thin facade over {@link Database.replacePageTemplates}; kept on
+     * `Archive` so the analyze pipeline can persist template keys without
+     * reaching into the low-level database class directly.
+     * @param templateKeysByUrl - Page URL → template key, as produced by
+     *   `@nitpicker/core`'s `classifyPageTemplates`.
+     */
+    replacePageTemplates(templateKeysByUrl: ReadonlyMap<string, string>): Promise<void>;
     /**
      * Promote previously-external pages that now fall under the (possibly extended)
      * scope back to a pending state so that the crawler re-scrapes them as fully

package/lib/archive/archive.js

@@ -286,6 +286,19 @@ export default class Archive extends ArchiveAccessor {
     async replaceAnalysisViolations(violations) {
         await this.#db.replaceAnalysisViolations(violations);
     }
+    /**
+     * Replaces the archive's DOM-structure template classification
+     * (`--templates`) with a fresh SQL-backed set.
+     *
+     * Thin facade over {@link Database.replacePageTemplates}; kept on
+     * `Archive` so the analyze pipeline can persist template keys without
+     * reaching into the low-level database class directly.
+     * @param templateKeysByUrl - Page URL → template key, as produced by
+     *   `@nitpicker/core`'s `classifyPageTemplates`.
+     */
+    async replacePageTemplates(templateKeysByUrl) {
+        await this.#db.replacePageTemplates(templateKeysByUrl);
+    }
     /**
      * Promote previously-external pages that now fall under the (possibly extended)
      * scope back to a pending state so that the crawler re-scrapes them as fully

package/lib/archive/create-adjunct-tables.d.ts

@@ -9,9 +9,16 @@ import type { Knex } from 'knex';
  *   an external link that failed DNS, or null for a process-level error)
  * - `page_tags` — Wappalyzer detections, FK → `content_items(id)`
  * - `page_jsonld` — JSON-LD / SpeculationRules, FK → `content_items(id)`
+ * - `page_main_content_headings` / `_images` / `_tables` / `_buttons` /
+ *   `_iframes` / `_videos` / `_audios` / `_canvases` — beholder
+ *   `MainContentsData` sub-entity arrays, one row per DOM element, FK →
+ *   `content_items(id)`
  * - `inventory_runs` — `--inventory` audit log (no FK; append-only)
  * - `analysis_text_refs` + `analysis_violations` — analyze-phase findings,
  *   FK → `content_items(id)`
+ * - `page_templates` — DOM-structure template classification (`--templates`,
+ *   `@nitpicker/core`'s `template-classification/`), one row per classified
+ *   page, FK → `content_items(id)`
  * - `page_html_blobs` + `page_html_ref` — content-addressable HTML
  *   snapshots, FK → `content_items(id)`
  *

package/lib/archive/create-adjunct-tables.js

@@ -8,9 +8,16 @@
  *   an external link that failed DNS, or null for a process-level error)
  * - `page_tags` — Wappalyzer detections, FK → `content_items(id)`
  * - `page_jsonld` — JSON-LD / SpeculationRules, FK → `content_items(id)`
+ * - `page_main_content_headings` / `_images` / `_tables` / `_buttons` /
+ *   `_iframes` / `_videos` / `_audios` / `_canvases` — beholder
+ *   `MainContentsData` sub-entity arrays, one row per DOM element, FK →
+ *   `content_items(id)`
  * - `inventory_runs` — `--inventory` audit log (no FK; append-only)
  * - `analysis_text_refs` + `analysis_violations` — analyze-phase findings,
  *   FK → `content_items(id)`
+ * - `page_templates` — DOM-structure template classification (`--templates`,
+ *   `@nitpicker/core`'s `template-classification/`), one row per classified
+ *   page, FK → `content_items(id)`
  * - `page_html_blobs` + `page_html_ref` — content-addressable HTML
  *   snapshots, FK → `content_items(id)`
  *
@@ -129,6 +136,134 @@ export async function createAdjunctTables(instance) {
         // `list_pages_by_jsonld_type` JOINs.
         await instance.raw('CREATE INDEX page_jsonld_type_pageId ON page_jsonld(type, pageId)');
     }
+    // Beholder `MainContentsData` sub-entities, one adjunct table per array
+    // (headings/images/tables/buttons/iframes/videos/audios/canvases). Same
+    // shape as `page_tags` / `page_jsonld`: `pageId` FK → `content_items(id)`
+    // ON DELETE CASCADE, individually guarded so any subset can pre-exist.
+    // `order` preserves the DOM traversal order beholder returns the array
+    // in (0-based); it is not itself an index target since these tables are
+    // always read whole-page via `WHERE pageId = ? ORDER BY "order"`.
+    if (!(await instance.schema.hasTable('page_main_content_headings'))) {
+        await instance.schema.createTable('page_main_content_headings', (t) => {
+            t.increments('id');
+            t.integer('pageId')
+                .notNullable()
+                .unsigned()
+                .references('content_items.id')
+                .onDelete('CASCADE');
+            t.integer('order').notNullable();
+            t.text('text');
+            t.integer('level').notNullable();
+            t.index('pageId');
+        });
+    }
+    if (!(await instance.schema.hasTable('page_main_content_images'))) {
+        await instance.schema.createTable('page_main_content_images', (t) => {
+            t.increments('id');
+            t.integer('pageId')
+                .notNullable()
+                .unsigned()
+                .references('content_items.id')
+                .onDelete('CASCADE');
+            t.integer('order').notNullable();
+            t.string('src', 8190).notNullable();
+            t.text('alt').notNullable();
+            t.index('pageId');
+        });
+    }
+    if (!(await instance.schema.hasTable('page_main_content_tables'))) {
+        await instance.schema.createTable('page_main_content_tables', (t) => {
+            t.increments('id');
+            t.integer('pageId')
+                .notNullable()
+                .unsigned()
+                .references('content_items.id')
+                .onDelete('CASCADE');
+            t.integer('order').notNullable();
+            t.integer('rows').notNullable();
+            t.integer('cols').notNullable();
+            t.boolean('hasHeader').notNullable();
+            t.boolean('hasFooter').notNullable();
+            t.boolean('hasMergedCell').notNullable();
+            t.index('pageId');
+        });
+    }
+    if (!(await instance.schema.hasTable('page_main_content_buttons'))) {
+        await instance.schema.createTable('page_main_content_buttons', (t) => {
+            t.increments('id');
+            t.integer('pageId')
+                .notNullable()
+                .unsigned()
+                .references('content_items.id')
+                .onDelete('CASCADE');
+            t.integer('order').notNullable();
+            t.string('nodeName').notNullable();
+            t.string('role');
+            t.string('type');
+            t.text('text');
+            t.boolean('disabled').notNullable();
+            t.index('pageId');
+        });
+    }
+    if (!(await instance.schema.hasTable('page_main_content_iframes'))) {
+        await instance.schema.createTable('page_main_content_iframes', (t) => {
+            t.increments('id');
+            t.integer('pageId')
+                .notNullable()
+                .unsigned()
+                .references('content_items.id')
+                .onDelete('CASCADE');
+            t.integer('order').notNullable();
+            t.string('src', 8190).notNullable();
+            t.text('title');
+            t.string('width');
+            t.string('height');
+            t.index('pageId');
+        });
+    }
+    if (!(await instance.schema.hasTable('page_main_content_videos'))) {
+        await instance.schema.createTable('page_main_content_videos', (t) => {
+            t.increments('id');
+            t.integer('pageId')
+                .notNullable()
+                .unsigned()
+                .references('content_items.id')
+                .onDelete('CASCADE');
+            t.integer('order').notNullable();
+            t.string('src', 8190).notNullable();
+            t.string('poster', 8190);
+            t.integer('width').notNullable();
+            t.integer('height').notNullable();
+            t.index('pageId');
+        });
+    }
+    if (!(await instance.schema.hasTable('page_main_content_audios'))) {
+        await instance.schema.createTable('page_main_content_audios', (t) => {
+            t.increments('id');
+            t.integer('pageId')
+                .notNullable()
+                .unsigned()
+                .references('content_items.id')
+                .onDelete('CASCADE');
+            t.integer('order').notNullable();
+            t.string('src', 8190).notNullable();
+            t.index('pageId');
+        });
+    }
+    if (!(await instance.schema.hasTable('page_main_content_canvases'))) {
+        await instance.schema.createTable('page_main_content_canvases', (t) => {
+            t.increments('id');
+            t.integer('pageId')
+                .notNullable()
+                .unsigned()
+                .references('content_items.id')
+                .onDelete('CASCADE');
+            t.integer('order').notNullable();
+            t.integer('width').notNullable();
+            t.integer('height').notNullable();
+            t.index('pageId');
+        });
+    }
     if (!(await instance.schema.hasTable('inventory_runs'))) {
         await instance.schema.createTable('inventory_runs', (t) => {
             // One row per successful `--inventory <list>` invocation. The
@@ -184,6 +319,21 @@ export async function createAdjunctTables(instance) {
         await instance.raw('CREATE INDEX av_code_order ON analysis_violations(code_sort_key, id)');
         await instance.raw('CREATE INDEX av_page ON analysis_violations(page_id, id)');
     }
+    // DOM-structure template classification (`--templates`). One row per
+    // internal HTML page that was classified; `page_id` is both the PK and
+    // the natural key (1:1 with `content_items`), so — unlike
+    // `analysis_violations`, which is 1:many and needs a surrogate `id` —
+    // there's nothing to index beyond the PK itself. `WITHOUT ROWID` packs
+    // rows directly in the PK b-tree, matching `page_html_ref`'s shape
+    // (small fixed-width row, PK-only lookups).
+    if (!(await instance.schema.hasTable('page_templates'))) {
+        await instance.raw(`
+			CREATE TABLE page_templates (
+				page_id      INTEGER PRIMARY KEY REFERENCES content_items(id),
+				template_key TEXT NOT NULL
+			) WITHOUT ROWID
+		`);
+    }
     // Content-addressable HTML blob storage. Knex's schema builder doesn't
     // expose a WITHOUT ROWID toggle, so the BLOB tables are created via raw
     // SQL. WITHOUT ROWID keeps the rows packed inside the b-tree leaves

package/lib/archive/create-entity-tables.d.ts

@@ -136,6 +136,20 @@ import type { Knex } from 'knex';
  * routing so the two entities never disagree on which values count as
  * "large data URI".
  *
+ * **`page_meta.main_content_*` / `scroll_height_*` columns.** Denormalised
+ * aggregates derived from beholder's `MainContentsData` / `ScrollHeightData`
+ * (word/element counts, desktop+mobile scroll height), following the same
+ * write-once-at-scrape-time pattern as `tag_count` / `jsonld_count` so list
+ * / detail reads never re-derive them from the per-page child tables
+ * (`page_main_content_headings` etc., see `create-adjunct-tables.ts`).
+ * `main_content_node_name` / `_id` / `_role` / `_selector` / `_class_list`
+ * identify the detected main-content element; unlike `title_text_id` /
+ * `description_text_id` these are stored as plain `TEXT` rather than routed
+ * through `text_refs` — the values are page-specific diagnostics with low
+ * cross-page reuse, so the ref-table dedup machinery would add write-path
+ * cost without a corresponding storage win. `main_content_class_list` holds
+ * a JSON-encoded string array.
+ *
  * ### Index rationale
  *
  * Every index below reflects a legacy-baseline single-column index that

package/lib/archive/create-entity-tables.js

@@ -135,6 +135,20 @@
  * routing so the two entities never disagree on which values count as
  * "large data URI".
  *
+ * **`page_meta.main_content_*` / `scroll_height_*` columns.** Denormalised
+ * aggregates derived from beholder's `MainContentsData` / `ScrollHeightData`
+ * (word/element counts, desktop+mobile scroll height), following the same
+ * write-once-at-scrape-time pattern as `tag_count` / `jsonld_count` so list
+ * / detail reads never re-derive them from the per-page child tables
+ * (`page_main_content_headings` etc., see `create-adjunct-tables.ts`).
+ * `main_content_node_name` / `_id` / `_role` / `_selector` / `_class_list`
+ * identify the detected main-content element; unlike `title_text_id` /
+ * `description_text_id` these are stored as plain `TEXT` rather than routed
+ * through `text_refs` — the values are page-specific diagnostics with low
+ * cross-page reuse, so the ref-table dedup machinery would add write-path
+ * cost without a corresponding storage win. `main_content_class_list` holds
+ * a JSON-encoded string array.
+ *
  * ### Index rationale
  *
  * Every index below reflects a legacy-baseline single-column index that
@@ -248,7 +262,24 @@ export async function createEntityTables(instance) {
 			tag_count                   INTEGER,
 			jsonld_count                INTEGER,
 			tags_providers_csv          TEXT,
-			meta_extras_json_id         INTEGER REFERENCES json_refs(id)
+			meta_extras_json_id         INTEGER REFERENCES json_refs(id),
+			main_content_node_name      TEXT,
+			main_content_id             TEXT,
+			main_content_role           TEXT,
+			main_content_selector       TEXT,
+			main_content_class_list     TEXT,
+			main_content_word_count     INTEGER,
+			main_content_body_word_count INTEGER,
+			main_content_heading_count  INTEGER,
+			main_content_image_count    INTEGER,
+			main_content_table_count    INTEGER,
+			main_content_button_count   INTEGER,
+			main_content_iframe_count   INTEGER,
+			main_content_video_count    INTEGER,
+			main_content_audio_count    INTEGER,
+			main_content_canvas_count   INTEGER,
+			scroll_height_desktop       INTEGER,
+			scroll_height_mobile        INTEGER
 		)
 	`);
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_page_meta_og_type ON page_meta(og_type)');

package/lib/archive/database.d.ts

@@ -1,4 +1,4 @@
-import type { JsonLdRow, TagRow } from './meta/types.js';
+import type { JsonLdRow, MainContentAudioRow, MainContentButtonRow, MainContentCanvasRow, MainContentHeadingRow, MainContentIframeRow, MainContentImageRow, MainContentTableRow, MainContentVideoRow, TagRow } from './meta/types.js';
 import type { Config, DatabaseOption, DB_Redirect, DB_Resource, DatabaseEvent, InventoryRunMeta, PageFilter, PageSource } from './types.js';
 import type { PageData, Resource } from '../utils/types/types.js';
 import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
@@ -46,6 +46,12 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @returns An array of anchor records with resolved URL, title, status, and content type.
      */
     getAnchorsOnPage(pageId: number): Promise<any[]>;
+    /**
+     * Retrieves all `page_main_content_audios` rows for the given page id.
+     * Delegates to {@link getAudiosOfPageOp}.
+     * @param pageId
+     */
+    getAudiosOfPage(pageId: number): Promise<MainContentAudioRow[]>;
     /**
      * Retrieves the base URL of the crawl session from the `info` table.
      * Delegates to {@link getBaseUrlOp}.
@@ -53,6 +59,18 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @throws {Error} If no base URL is found in the database.
      */
     getBaseUrl(): Promise<string>;
+    /**
+     * Retrieves all `page_main_content_buttons` rows for the given page id.
+     * Delegates to {@link getButtonsOfPageOp}.
+     * @param pageId
+     */
+    getButtonsOfPage(pageId: number): Promise<MainContentButtonRow[]>;
+    /**
+     * Retrieves all `page_main_content_canvases` rows for the given page id.
+     * Delegates to {@link getCanvasesOfPageOp}.
+     * @param pageId
+     */
+    getCanvasesOfPage(pageId: number): Promise<MainContentCanvasRow[]>;
     /**
      * Retrieves the full crawl configuration from the `info` table.
      * Delegates to {@link getConfigOp}.
@@ -85,6 +103,12 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @returns URLs found in `resources`.
      */
     getExistingResourceUrls(urls: readonly string[]): Promise<string[]>;
+    /**
+     * Retrieves all `page_main_content_headings` rows for the given page id.
+     * Delegates to {@link getHeadingsOfPageOp}.
+     * @param pageId
+     */
+    getHeadingsOfPage(pageId: number): Promise<MainContentHeadingRow[]>;
     /**
      * Reads the HTML snapshot stored as a zstd-compressed BLOB for the given page.
      * Delegates to {@link getHtmlOfPageByIdOp}.
@@ -92,6 +116,12 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @returns The decompressed HTML string, or `null` if no snapshot is stored.
      */
     getHtmlOfPageById(pageId: number): Promise<string | null>;
+    /**
+     * Retrieves all `page_main_content_iframes` rows for the given page id.
+     * Delegates to {@link getIframesOfPageOp}.
+     * @param pageId
+     */
+    getIframesOfPage(pageId: number): Promise<MainContentIframeRow[]>;
     /**
      * Retrieves all `page_jsonld` rows for the given page id, parsed back into
      * {@link JsonLdRow} shape. Delegates to {@link getJsonLdOfPageOp}.
@@ -105,6 +135,18 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @returns The Knex instance connected to the SQLite database.
      */
     getKnex(): Knex;
+    /**
+     * Retrieves all `page_main_content_images` rows for the given page id.
+     * Delegates to {@link getMainContentImagesOfPageOp}.
+     * @param pageId
+     */
+    getMainContentImagesOfPage(pageId: number): Promise<MainContentImageRow[]>;
+    /**
+     * Retrieves all `page_main_content_tables` rows for the given page id.
+     * Delegates to {@link getMainContentTablesOfPageOp}.
+     * @param pageId
+     */
+    getMainContentTablesOfPage(pageId: number): Promise<MainContentTableRow[]>;
     /**
      * Retrieves the crawl session name from the `info` table.
      * Delegates to {@link getNameOp}.
@@ -207,6 +249,12 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @param pageId
      */
     getTagsOfPage(pageId: number): Promise<TagRow[]>;
+    /**
+     * Retrieves all `page_main_content_videos` rows for the given page id.
+     * Delegates to {@link getVideosOfPageOp}.
+     * @param pageId
+     */
+    getVideosOfPage(pageId: number): Promise<MainContentVideoRow[]>;
     /**
      * Records a crawler-level (`error` channel) failure into `crawl_errors`.
      * Delegates to {@link insertCrawlErrorOp}.
@@ -291,6 +339,12 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
         line?: number | null;
         col?: number | null;
     }[]): Promise<void>;
+    /**
+     * Replaces the stored DOM-structure template classification with a
+     * freshly generated set. Delegates to {@link replacePageTemplatesOp}.
+     * @param templateKeysByUrl - Page URL → template key.
+     */
+    replacePageTemplates(templateKeysByUrl: ReadonlyMap<string, string>): Promise<void>;
     /**
      * Promote previously-external pages whose URL falls under any of the new
      * scope entries back to a "needs scraping" state.