@nitpicker/crawler 0.13.0 → 0.14.0

package/lib/archive/meta/types.d.ts

@@ -93,6 +93,269 @@ export interface PageDenormalizedColumns {
     /** Sorted unique providers, comma-separated (empty string when no tags). */
     tags_providers_csv: string;
 }
+/**
+ * Denormalised aggregates computed at write time from beholder's
+ * `MainContentsData` / `ScrollHeightData`.
+ *
+ * Stored on `page_meta` following the same pattern as
+ * {@link PageDenormalizedColumns} (`tag_count` / `jsonld_count`): the full
+ * per-element detail lives in the `page_main_content_*` child tables, while
+ * these scalar columns let list / detail reads answer "how many headings?"
+ * without joining them. All fields are `null` when the page was not fully
+ * rendered (external / non-HTML / metadata-only scrape) — see
+ * `compute-main-contents-denormalized.ts` for the `null`-in-null-out contract.
+ * @see compute-main-contents-denormalized.ts
+ * @example
+ * const denorm: MainContentsDenormalizedColumns = {
+ *   main_content_node_name: 'MAIN',
+ *   main_content_id: null,
+ *   main_content_role: null,
+ *   main_content_selector: 'main.l-main',
+ *   main_content_class_list: '["l-main"]',
+ *   main_content_word_count: 1240,
+ *   main_content_body_word_count: 1580,
+ *   main_content_heading_count: 6,
+ *   main_content_image_count: 3,
+ *   main_content_table_count: 0,
+ *   main_content_button_count: 1,
+ *   main_content_iframe_count: 0,
+ *   main_content_video_count: 0,
+ *   main_content_audio_count: 0,
+ *   main_content_canvas_count: 0,
+ *   scroll_height_desktop: 3200,
+ *   scroll_height_mobile: 5400,
+ * };
+ */
+export interface MainContentsDenormalizedColumns {
+    /** Detected main-content element's `nodeName` (e.g. `'MAIN'`), or `null`. */
+    main_content_node_name: string | null;
+    /** Detected main-content element's `id`, or `null`. */
+    main_content_id: string | null;
+    /** Detected main-content element's `role` attribute, or `null`. */
+    main_content_role: string | null;
+    /** Diagnostic tag+id+class selector for the detected element, or `null`. */
+    main_content_selector: string | null;
+    /** JSON-encoded array of the detected element's CSS classes, or `null`. */
+    main_content_class_list: string | null;
+    /** Character count of the main region's text content, or `null`. */
+    main_content_word_count: number | null;
+    /** Character count of `document.body`'s text content, or `null`. */
+    main_content_body_word_count: number | null;
+    /** Number of headings within the main region, or `null`. */
+    main_content_heading_count: number | null;
+    /** Number of images within the main region, or `null`. */
+    main_content_image_count: number | null;
+    /** Number of tables within the main region, or `null`. */
+    main_content_table_count: number | null;
+    /** Number of button-like elements within the main region, or `null`. */
+    main_content_button_count: number | null;
+    /** Number of iframes within the main region, or `null`. */
+    main_content_iframe_count: number | null;
+    /** Number of videos within the main region, or `null`. */
+    main_content_video_count: number | null;
+    /** Number of audios within the main region, or `null`. */
+    main_content_audio_count: number | null;
+    /** Number of canvases within the main region, or `null`. */
+    main_content_canvas_count: number | null;
+    /** `document.body.scrollHeight` at the desktop-compact preset, or `null`. */
+    scroll_height_desktop: number | null;
+    /** `document.body.scrollHeight` at the mobile-small preset, or `null`. */
+    scroll_height_mobile: number | null;
+}
+/**
+ * One row in the `page_main_content_headings` table.
+ * @example
+ * const row: MainContentHeadingRow = { id: 1, pageId: 42, order: 0, text: 'Welcome', level: 1 };
+ */
+export interface MainContentHeadingRow {
+    /** Auto-increment primary key. */
+    id: number;
+    /** FK to `content_items.id`. */
+    pageId: number;
+    /** 0-based DOM traversal order within the main content region. */
+    order: number;
+    /** Heading text after whitespace removal, or `null` when empty. */
+    text: string | null;
+    /** Heading level (1-6) from the tag name. */
+    level: 1 | 2 | 3 | 4 | 5 | 6;
+}
+/**
+ * One row in the `page_main_content_images` table.
+ * @example
+ * const row: MainContentImageRow = {
+ *   id: 1,
+ *   pageId: 42,
+ *   order: 0,
+ *   src: 'https://example.com/a.png',
+ *   alt: 'A photo',
+ * };
+ */
+export interface MainContentImageRow {
+    /** Auto-increment primary key. */
+    id: number;
+    /** FK to `content_items.id`. */
+    pageId: number;
+    /** 0-based DOM traversal order within the main content region. */
+    order: number;
+    /** Resolved absolute `src` URL. */
+    src: string;
+    /** `alt` attribute value (may be an empty string). */
+    alt: string;
+}
+/**
+ * One row in the `page_main_content_tables` table.
+ * @example
+ * const row: MainContentTableRow = {
+ *   id: 1,
+ *   pageId: 42,
+ *   order: 0,
+ *   rows: 3,
+ *   cols: 4,
+ *   hasHeader: 1,
+ *   hasFooter: 0,
+ *   hasMergedCell: 0,
+ * };
+ */
+export interface MainContentTableRow {
+    /** Auto-increment primary key. */
+    id: number;
+    /** FK to `content_items.id`. */
+    pageId: number;
+    /** 0-based DOM traversal order within the main content region. */
+    order: number;
+    /** Number of `<tr>` elements. */
+    rows: number;
+    /** Number of `th`/`td` cells in the first row. */
+    cols: number;
+    /** Whether the table contains a `<thead>` (raw SQLite 0/1; knex does not round-trip `.boolean()` columns back to JS `boolean` on read). */
+    hasHeader: 0 | 1;
+    /** Whether the table contains a `<tfoot>` (raw SQLite 0/1). */
+    hasFooter: 0 | 1;
+    /** Whether any cell uses `colspan` or `rowspan` (raw SQLite 0/1). */
+    hasMergedCell: 0 | 1;
+}
+/**
+ * One row in the `page_main_content_buttons` table.
+ * @example
+ * const row: MainContentButtonRow = {
+ *   id: 1,
+ *   pageId: 42,
+ *   order: 0,
+ *   nodeName: 'BUTTON',
+ *   role: null,
+ *   type: 'submit',
+ *   text: 'Send',
+ *   disabled: 0,
+ * };
+ */
+export interface MainContentButtonRow {
+    /** Auto-increment primary key. */
+    id: number;
+    /** FK to `content_items.id`. */
+    pageId: number;
+    /** 0-based DOM traversal order within the main content region. */
+    order: number;
+    /** Element tag name (e.g. `'BUTTON'`, `'A'`, `'DIV'`). */
+    nodeName: string;
+    /** `role` attribute, or `null` when absent. */
+    role: string | null;
+    /** `type` for `<button>` / `<input>`, otherwise `null`. */
+    type: string | null;
+    /** Label text after whitespace removal, or `null` when empty. */
+    text: string | null;
+    /** `true` when `disabled` or `aria-disabled="true"` (raw SQLite 0/1). */
+    disabled: 0 | 1;
+}
+/**
+ * One row in the `page_main_content_iframes` table.
+ * @example
+ * const row: MainContentIframeRow = {
+ *   id: 1,
+ *   pageId: 42,
+ *   order: 0,
+ *   src: 'https://example.com/embed',
+ *   title: null,
+ *   width: '640',
+ *   height: '360',
+ * };
+ */
+export interface MainContentIframeRow {
+    /** Auto-increment primary key. */
+    id: number;
+    /** FK to `content_items.id`. */
+    pageId: number;
+    /** 0-based DOM traversal order within the main content region. */
+    order: number;
+    /** Resolved absolute `src` URL. */
+    src: string;
+    /** `title` attribute, or `null` when absent. */
+    title: string | null;
+    /** Raw `width` attribute string, or `null` when absent. */
+    width: string | null;
+    /** Raw `height` attribute string, or `null` when absent. */
+    height: string | null;
+}
+/**
+ * One row in the `page_main_content_videos` table.
+ * @example
+ * const row: MainContentVideoRow = {
+ *   id: 1,
+ *   pageId: 42,
+ *   order: 0,
+ *   src: 'https://example.com/v.mp4',
+ *   poster: null,
+ *   width: 640,
+ *   height: 360,
+ * };
+ */
+export interface MainContentVideoRow {
+    /** Auto-increment primary key. */
+    id: number;
+    /** FK to `content_items.id`. */
+    pageId: number;
+    /** 0-based DOM traversal order within the main content region. */
+    order: number;
+    /** Resolved media URL. */
+    src: string;
+    /** Resolved `poster` URL, or `null` when unset. */
+    poster: string | null;
+    /** IDL `width` in pixels. */
+    width: number;
+    /** IDL `height` in pixels. */
+    height: number;
+}
+/**
+ * One row in the `page_main_content_audios` table.
+ * @example
+ * const row: MainContentAudioRow = { id: 1, pageId: 42, order: 0, src: 'https://example.com/a.mp3' };
+ */
+export interface MainContentAudioRow {
+    /** Auto-increment primary key. */
+    id: number;
+    /** FK to `content_items.id`. */
+    pageId: number;
+    /** 0-based DOM traversal order within the main content region. */
+    order: number;
+    /** Resolved media URL. */
+    src: string;
+}
+/**
+ * One row in the `page_main_content_canvases` table.
+ * @example
+ * const row: MainContentCanvasRow = { id: 1, pageId: 42, order: 0, width: 300, height: 150 };
+ */
+export interface MainContentCanvasRow {
+    /** Auto-increment primary key. */
+    id: number;
+    /** FK to `content_items.id`. */
+    pageId: number;
+    /** 0-based DOM traversal order within the main content region. */
+    order: number;
+    /** IDL bitmap width. */
+    width: number;
+    /** IDL bitmap height. */
+    height: number;
+}
 /**
  * One row in the `page_jsonld` table.
  *

package/lib/archive/migrate-info-main-content-selector.d.ts

@@ -0,0 +1,11 @@
+import type { Knex } from 'knex';
+/**
+ * Adds the `info.mainContentSelector` column to archives created before it
+ * existed. `CREATE TABLE IF NOT EXISTS` (used for `info` itself) cannot
+ * retrofit a new column onto an already-existing table, so this lightweight,
+ * `hasColumn`-guarded `ALTER TABLE` runs on every `initSchema` call —
+ * idempotent, and self-healing for archives whose provisioning crashed
+ * partway through.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function migrateInfoMainContentSelector(instance: Knex): Promise<void>;

package/lib/archive/migrate-info-main-content-selector.js

@@ -0,0 +1,24 @@
+/**
+ * Adds the `info.mainContentSelector` column to archives created before it
+ * existed. `CREATE TABLE IF NOT EXISTS` (used for `info` itself) cannot
+ * retrofit a new column onto an already-existing table, so this lightweight,
+ * `hasColumn`-guarded `ALTER TABLE` runs on every `initSchema` call —
+ * idempotent, and self-healing for archives whose provisioning crashed
+ * partway through.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function migrateInfoMainContentSelector(instance) {
+    const hasInfo = await instance.schema.hasTable('info');
+    if (!hasInfo) {
+        return;
+    }
+    const hasColumn = await instance.schema.hasColumn('info', 'mainContentSelector');
+    if (hasColumn) {
+        return;
+    }
+    await instance.schema.table('info', (t) => {
+        t.string('mainContentSelector');
+    });
+    // eslint-disable-next-line no-console
+    console.error('[migrate] info.mainContentSelector column added');
+}

package/lib/archive/migrate-main-contents-columns.d.ts

@@ -0,0 +1,21 @@
+import type { Knex } from 'knex';
+/**
+ * Add the beholder `MainContentsData` / `ScrollHeightData` denormalised
+ * aggregate columns to `page_meta` on archives created before this feature.
+ *
+ * `page_meta` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `page_meta` predates this change. This mirrors
+ * {@link import('./migrate-info-roots.js').migrateInfoRoots}'s `info.roots`
+ * catch-up: a `hasColumn`-guarded `ALTER TABLE` for the one column set that
+ * `CREATE TABLE IF NOT EXISTS` cannot retrofit.
+ *
+ * Idempotent: a no-op once the columns exist. Guards on `page_meta`'s
+ * existence defensively, though by the time this runs (after `initSchema`,
+ * itself after `assertCompatibleVersion` rejects pre-0.13 archives) the
+ * table is always present.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function migrateMainContentsColumns(instance: Knex): Promise<void>;

package/lib/archive/migrate-main-contents-columns.js

@@ -0,0 +1,50 @@
+/**
+ * Add the beholder `MainContentsData` / `ScrollHeightData` denormalised
+ * aggregate columns to `page_meta` on archives created before this feature.
+ *
+ * `page_meta` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `page_meta` predates this change. This mirrors
+ * {@link import('./migrate-info-roots.js').migrateInfoRoots}'s `info.roots`
+ * catch-up: a `hasColumn`-guarded `ALTER TABLE` for the one column set that
+ * `CREATE TABLE IF NOT EXISTS` cannot retrofit.
+ *
+ * Idempotent: a no-op once the columns exist. Guards on `page_meta`'s
+ * existence defensively, though by the time this runs (after `initSchema`,
+ * itself after `assertCompatibleVersion` rejects pre-0.13 archives) the
+ * table is always present.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function migrateMainContentsColumns(instance) {
+    const hasPageMeta = await instance.schema.hasTable('page_meta');
+    if (!hasPageMeta) {
+        return;
+    }
+    const hasColumn = await instance.schema.hasColumn('page_meta', 'main_content_word_count');
+    if (hasColumn) {
+        return;
+    }
+    await instance.schema.table('page_meta', (t) => {
+        t.string('main_content_node_name');
+        t.string('main_content_id');
+        t.string('main_content_role');
+        t.string('main_content_selector');
+        t.text('main_content_class_list');
+        t.integer('main_content_word_count');
+        t.integer('main_content_body_word_count');
+        t.integer('main_content_heading_count');
+        t.integer('main_content_image_count');
+        t.integer('main_content_table_count');
+        t.integer('main_content_button_count');
+        t.integer('main_content_iframe_count');
+        t.integer('main_content_video_count');
+        t.integer('main_content_audio_count');
+        t.integer('main_content_canvas_count');
+        t.integer('scroll_height_desktop');
+        t.integer('scroll_height_mobile');
+    });
+    // eslint-disable-next-line no-console
+    console.error('[migrate] page_meta.main_content_* / scroll_height_* columns added');
+}

package/lib/archive/page.d.ts

@@ -9,7 +9,7 @@ import type { Anchor, Redirect, Referrer, DB_Anchor, DB_Page, DB_Redirect, DB_Re
  * Keep in sync with {@link import('./meta/types.js').FlatPageMetaColumns} —
  * one row of `pages` has the same shape.
  */
-declare const FLAT_META_COLUMNS: readonly ["lang", "dir", "charset", "baseHref", "viewport_raw", "themeColor", "applicationName", "author", "generator", "publisher", "title", "description", "keywords", "robots_raw", "robots_noindex", "robots_nofollow", "robots_noarchive", "robots_noimageindex", "googlebot", "canonical", "amphtml", "manifest", "icon_href", "appleTouchIcon_href", "og_type", "og_title", "og_url", "og_site_name", "og_description", "og_image", "og_image_alt", "og_image_width", "og_image_height", "og_locale", "og_article_published_time", "og_article_modified_time", "twitter_card", "twitter_site", "twitter_creator", "twitter_title", "twitter_description", "twitter_image", "fb_app_id", "verification_google", "formatDetection_telephone", "tag_count", "jsonld_count", "tags_providers_csv"];
+declare const FLAT_META_COLUMNS: readonly ["lang", "dir", "charset", "baseHref", "viewport_raw", "themeColor", "applicationName", "author", "generator", "publisher", "title", "description", "keywords", "robots_raw", "robots_noindex", "robots_nofollow", "robots_noarchive", "robots_noimageindex", "googlebot", "canonical", "amphtml", "manifest", "icon_href", "appleTouchIcon_href", "og_type", "og_title", "og_url", "og_site_name", "og_description", "og_image", "og_image_alt", "og_image_width", "og_image_height", "og_locale", "og_article_published_time", "og_article_modified_time", "twitter_card", "twitter_site", "twitter_creator", "twitter_title", "twitter_description", "twitter_image", "fb_app_id", "verification_google", "formatDetection_telephone", "tag_count", "jsonld_count", "tags_providers_csv", "main_content_node_name", "main_content_id", "main_content_role", "main_content_selector", "main_content_class_list", "main_content_word_count", "main_content_body_word_count", "main_content_heading_count", "main_content_image_count", "main_content_table_count", "main_content_button_count", "main_content_iframe_count", "main_content_video_count", "main_content_audio_count", "main_content_canvas_count", "scroll_height_desktop", "scroll_height_mobile"];
 /**
  * Represents a crawled page stored in the archive.
  *
@@ -90,6 +90,71 @@ export default class Page {
      * or null for legacy rows / never-scraped pages.
      */
     get lastCrawledAt(): number | null;
+    /**
+     * Detected main-content element's `nodeName` (e.g. `'MAIN'`), or null when
+     * no main region was found or the page was not fully rendered.
+     */
+    get mainContentNodeName(): string | null;
+    /**
+     * Detected main-content element's `id`, or null.
+     */
+    get mainContentId(): string | null;
+    /**
+     * Detected main-content element's `role` attribute, or null.
+     */
+    get mainContentRole(): string | null;
+    /**
+     * Diagnostic tag+id+class selector for the detected main-content element, or null.
+     */
+    get mainContentSelector(): string | null;
+    /**
+     * Detected main-content element's CSS classes, or null when no main region
+     * was found. Parsed from the JSON-encoded `main_content_class_list` column.
+     */
+    get mainContentClassList(): string[] | null;
+    /**
+     * Character count of the main region's text content (denormalised
+     * aggregate written at scrape time), or null.
+     */
+    get mainContentWordCount(): number | null;
+    /**
+     * Character count of `document.body`'s text content (denormalised
+     * aggregate written at scrape time), or null.
+     */
+    get mainContentBodyWordCount(): number | null;
+    /**
+     * Number of headings within the main region (denormalised aggregate), or null.
+     */
+    get mainContentHeadingCount(): number | null;
+    /**
+     * Number of images within the main region (denormalised aggregate), or null.
+     */
+    get mainContentImageCount(): number | null;
+    /**
+     * Number of tables within the main region (denormalised aggregate), or null.
+     */
+    get mainContentTableCount(): number | null;
+    /**
+     * Number of button-like elements within the main region (denormalised
+     * aggregate), or null.
+     */
+    get mainContentButtonCount(): number | null;
+    /**
+     * Number of iframes within the main region (denormalised aggregate), or null.
+     */
+    get mainContentIframeCount(): number | null;
+    /**
+     * Number of videos within the main region (denormalised aggregate), or null.
+     */
+    get mainContentVideoCount(): number | null;
+    /**
+     * Number of audios within the main region (denormalised aggregate), or null.
+     */
+    get mainContentAudioCount(): number | null;
+    /**
+     * Number of canvases within the main region (denormalised aggregate), or null.
+     */
+    get mainContentCanvasCount(): number | null;
     /**
      * Iterable view over every flat meta column (~47 fields). Returns a frozen
      * record so consumers can pick fields by name without re-enumerating
@@ -157,6 +222,16 @@ export default class Page {
      * Use for diagnostics; specific directive flags live on `robots_*` getters.
      */
     get robots_raw(): string | null;
+    /**
+     * `document.body.scrollHeight` at the desktop-compact preset (denormalised
+     * aggregate written at scrape time), or null.
+     */
+    get scrollHeightDesktop(): number | null;
+    /**
+     * `document.body.scrollHeight` at the mobile-small preset (denormalised
+     * aggregate written at scrape time), or null.
+     */
+    get scrollHeightMobile(): number | null;
     /**
      * The reason this page was skipped during crawling, or null if it was not skipped.
      */
@@ -210,6 +285,31 @@ export default class Page {
      * @returns An array of {@link Anchor} objects representing the links on this page.
      */
     getAnchors(): Promise<Anchor[]>;
+    /**
+     * Retrieves the audios within this page's detected main content region
+     * from `page_main_content_audios`. Lazy — runs a single SELECT per call.
+     * @returns Ordered audio rows.
+     */
+    getAudios(): Promise<import("./meta/types.js").MainContentAudioRow[]>;
+    /**
+     * Retrieves the button-like elements within this page's detected main
+     * content region from `page_main_content_buttons`. Lazy — runs a single
+     * SELECT per call.
+     * @returns Ordered button rows.
+     */
+    getButtons(): Promise<import("./meta/types.js").MainContentButtonRow[]>;
+    /**
+     * Retrieves the canvases within this page's detected main content region
+     * from `page_main_content_canvases`. Lazy — runs a single SELECT per call.
+     * @returns Ordered canvas rows.
+     */
+    getCanvases(): Promise<import("./meta/types.js").MainContentCanvasRow[]>;
+    /**
+     * Retrieves the headings within this page's detected main content region
+     * from `page_main_content_headings`. Lazy — runs a single SELECT per call.
+     * @returns Ordered heading rows.
+     */
+    getHeadings(): Promise<import("./meta/types.js").MainContentHeadingRow[]>;
     /**
      * Thin wrapper that forwards this page's id to the accessor's
      * BLOB-read path. Lets callers hold a `Page` reference and ask for its
@@ -223,6 +323,12 @@ export default class Page {
      * }
      */
     getHtml(): Promise<string | null>;
+    /**
+     * Retrieves the iframes within this page's detected main content region
+     * from `page_main_content_iframes`. Lazy — runs a single SELECT per call.
+     * @returns Ordered iframe rows.
+     */
+    getIframes(): Promise<import("./meta/types.js").MainContentIframeRow[]>;
     /**
      * Retrieves the JSON-LD entries for this page from `page_jsonld`.
      * Lazy — runs a single SELECT per call. Returns entries in insertion
@@ -230,6 +336,19 @@ export default class Page {
      * @returns Ordered JSON-LD / SpeculationRules rows.
      */
     getJsonLd(): Promise<readonly JsonLdRow[]>;
+    /**
+     * Retrieves the images within this page's detected main content region
+     * from `page_main_content_images`. Lazy — runs a single SELECT per call.
+     * Distinct from the whole-page image scan (`image_items`).
+     * @returns Ordered image rows.
+     */
+    getMainContentImages(): Promise<import("./meta/types.js").MainContentImageRow[]>;
+    /**
+     * Retrieves the tables within this page's detected main content region
+     * from `page_main_content_tables`. Lazy — runs a single SELECT per call.
+     * @returns Ordered table rows.
+     */
+    getMainContentTables(): Promise<import("./meta/types.js").MainContentTableRow[]>;
     /**
      * Retrieves the referrers (incoming links) pointing to this page.
      * Uses pre-loaded data if available, otherwise queries the database.
@@ -248,6 +367,12 @@ export default class Page {
      * @returns Ordered tag rows.
      */
     getTags(): Promise<readonly TagRow[]>;
+    /**
+     * Retrieves the videos within this page's detected main content region
+     * from `page_main_content_videos`. Lazy — runs a single SELECT per call.
+     * @returns Ordered video rows.
+     */
+    getVideos(): Promise<import("./meta/types.js").MainContentVideoRow[]>;
     /**
      * Checks whether this page is an internal HTML page (not external and has `text/html` content type).
      * @returns `true` if this is an internal HTML page, `false` otherwise.
@@ -281,6 +406,24 @@ export default class Page {
         isInternalPage: boolean;
         getAnchors: Anchor[];
         getReferrers: Referrer[];
+        title: string | number | null;
+        main_content_word_count: string | number | null;
+        main_content_node_name: string | number | null;
+        main_content_id: string | number | null;
+        main_content_role: string | number | null;
+        main_content_selector: string | number | null;
+        main_content_class_list: string | number | null;
+        main_content_body_word_count: string | number | null;
+        main_content_heading_count: string | number | null;
+        main_content_image_count: string | number | null;
+        main_content_table_count: string | number | null;
+        main_content_button_count: string | number | null;
+        main_content_iframe_count: string | number | null;
+        main_content_video_count: string | number | null;
+        main_content_audio_count: string | number | null;
+        main_content_canvas_count: string | number | null;
+        scroll_height_desktop: string | number | null;
+        scroll_height_mobile: string | number | null;
         lang: string | number | null;
         dir: string | number | null;
         charset: string | number | null;
@@ -291,7 +434,6 @@ export default class Page {
         author: string | number | null;
         generator: string | number | null;
         publisher: string | number | null;
-        title: string | number | null;
         description: string | number | null;
         keywords: string | number | null;
         robots_raw: string | number | null;