@nitpicker/crawler 0.13.0 → 0.15.0

package/lib/archive/migrate-page-meta-body-hash.js

@@ -0,0 +1,52 @@
+/**
+ * Adds the `page_meta.body_hash` column to archives created before this
+ * feature, then ensures its index exists.
+ *
+ * `page_meta` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `page_meta` predates this change. This mirrors
+ * {@link import('./migrate-main-contents-columns.js').migrateMainContentsColumns}'s
+ * catch-up: a `hasColumn`-guarded `ALTER TABLE` for the one column
+ * `CREATE TABLE IF NOT EXISTS` cannot retrofit.
+ *
+ * The index is created here — unconditionally, after the column-add guard,
+ * not inside `createEntityTables`'s DDL — for both a fresh archive (where
+ * `body_hash` already exists from the DDL, so only the index still needs
+ * creating) and a legacy archive (where the column is added just above,
+ * then the index follows in the same call). `createEntityTables` runs
+ * unconditionally on every archive open, including legacy archives that
+ * still lack `body_hash` at that point; an unconditional
+ * `CREATE INDEX ... body_hash` there would fail with `no such column` before
+ * this migration ever runs. This function is the one place guaranteed to
+ * run only after the column is confirmed present, for both archive kinds.
+ *
+ * Only adds the column and its index — it does not backfill values for
+ * existing rows (they stay `NULL`). That backfill runs separately, from
+ * `backfillBodyHashFromHtmlBlobs` during a viewer-read-model build, since it
+ * requires decompressing every page's stored HTML and is too heavy to run on
+ * every archive open.
+ *
+ * Idempotent: adding the column is a no-op once it exists (the index
+ * creation always runs, but `IF NOT EXISTS` makes repeat runs a no-op too).
+ * Guards on `page_meta`'s existence defensively, though by the time this
+ * runs (after `initSchema`, itself after `assertCompatibleVersion` rejects
+ * pre-0.13 archives) the table is always present.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function migratePageMetaBodyHash(instance) {
+    const hasPageMeta = await instance.schema.hasTable('page_meta');
+    if (!hasPageMeta) {
+        return;
+    }
+    const hasColumn = await instance.schema.hasColumn('page_meta', 'body_hash');
+    if (!hasColumn) {
+        await instance.schema.table('page_meta', (t) => {
+            t.binary('body_hash');
+        });
+        // eslint-disable-next-line no-console
+        console.error('[migrate] page_meta.body_hash column added');
+    }
+    await instance.raw('CREATE INDEX IF NOT EXISTS idx_page_meta_body_hash ON page_meta(body_hash)');
+}

package/lib/archive/migrate-page-meta-console-error-count.d.ts

@@ -0,0 +1,22 @@
+import type { Knex } from 'knex';
+/**
+ * Adds the `page_meta.console_error_count` column to archives created
+ * before this feature (issue #228).
+ *
+ * `page_meta` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `page_meta` predates this change. Same
+ * catch-up shape as {@link import('./migrate-page-meta-body-hash.js').migratePageMetaBodyHash}.
+ *
+ * Unlike `body_hash`, there is no separate backfill step: a page that has
+ * never been re-scraped since this feature shipped has no
+ * `page_console_logs` rows either, so `0` (the value `replaceConsoleLogs`
+ * writes for a page with no error/pageerror entries) is already the
+ * correct value, not a placeholder pending backfill.
+ *
+ * Idempotent: adding the column is a no-op once it exists.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function migratePageMetaConsoleErrorCount(instance: Knex): Promise<void>;

package/lib/archive/migrate-page-meta-console-error-count.js

@@ -0,0 +1,34 @@
+/**
+ * Adds the `page_meta.console_error_count` column to archives created
+ * before this feature (issue #228).
+ *
+ * `page_meta` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `page_meta` predates this change. Same
+ * catch-up shape as {@link import('./migrate-page-meta-body-hash.js').migratePageMetaBodyHash}.
+ *
+ * Unlike `body_hash`, there is no separate backfill step: a page that has
+ * never been re-scraped since this feature shipped has no
+ * `page_console_logs` rows either, so `0` (the value `replaceConsoleLogs`
+ * writes for a page with no error/pageerror entries) is already the
+ * correct value, not a placeholder pending backfill.
+ *
+ * Idempotent: adding the column is a no-op once it exists.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function migratePageMetaConsoleErrorCount(instance) {
+    const hasPageMeta = await instance.schema.hasTable('page_meta');
+    if (!hasPageMeta) {
+        return;
+    }
+    const hasColumn = await instance.schema.hasColumn('page_meta', 'console_error_count');
+    if (!hasColumn) {
+        await instance.schema.table('page_meta', (t) => {
+            t.integer('console_error_count');
+        });
+        // eslint-disable-next-line no-console
+        console.error('[migrate] page_meta.console_error_count column added');
+    }
+}

package/lib/archive/page.d.ts

@@ -9,7 +9,7 @@ import type { Anchor, Redirect, Referrer, DB_Anchor, DB_Page, DB_Redirect, DB_Re
  * Keep in sync with {@link import('./meta/types.js').FlatPageMetaColumns} —
  * one row of `pages` has the same shape.
  */
-declare const FLAT_META_COLUMNS: readonly ["lang", "dir", "charset", "baseHref", "viewport_raw", "themeColor", "applicationName", "author", "generator", "publisher", "title", "description", "keywords", "robots_raw", "robots_noindex", "robots_nofollow", "robots_noarchive", "robots_noimageindex", "googlebot", "canonical", "amphtml", "manifest", "icon_href", "appleTouchIcon_href", "og_type", "og_title", "og_url", "og_site_name", "og_description", "og_image", "og_image_alt", "og_image_width", "og_image_height", "og_locale", "og_article_published_time", "og_article_modified_time", "twitter_card", "twitter_site", "twitter_creator", "twitter_title", "twitter_description", "twitter_image", "fb_app_id", "verification_google", "formatDetection_telephone", "tag_count", "jsonld_count", "tags_providers_csv"];
+declare const FLAT_META_COLUMNS: readonly ["lang", "dir", "charset", "baseHref", "viewport_raw", "themeColor", "applicationName", "author", "generator", "publisher", "title", "description", "keywords", "robots_raw", "robots_noindex", "robots_nofollow", "robots_noarchive", "robots_noimageindex", "googlebot", "canonical", "amphtml", "manifest", "icon_href", "appleTouchIcon_href", "og_type", "og_title", "og_url", "og_site_name", "og_description", "og_image", "og_image_alt", "og_image_width", "og_image_height", "og_locale", "og_article_published_time", "og_article_modified_time", "twitter_card", "twitter_site", "twitter_creator", "twitter_title", "twitter_description", "twitter_image", "fb_app_id", "verification_google", "formatDetection_telephone", "tag_count", "jsonld_count", "tags_providers_csv", "main_content_node_name", "main_content_id", "main_content_role", "main_content_selector", "main_content_class_list", "main_content_word_count", "main_content_body_word_count", "main_content_heading_count", "main_content_image_count", "main_content_table_count", "main_content_button_count", "main_content_iframe_count", "main_content_video_count", "main_content_audio_count", "main_content_canvas_count", "scroll_height_desktop", "scroll_height_mobile"];
 /**
  * Represents a crawled page stored in the archive.
  *
@@ -90,6 +90,71 @@ export default class Page {
      * or null for legacy rows / never-scraped pages.
      */
     get lastCrawledAt(): number | null;
+    /**
+     * Detected main-content element's `nodeName` (e.g. `'MAIN'`), or null when
+     * no main region was found or the page was not fully rendered.
+     */
+    get mainContentNodeName(): string | null;
+    /**
+     * Detected main-content element's `id`, or null.
+     */
+    get mainContentId(): string | null;
+    /**
+     * Detected main-content element's `role` attribute, or null.
+     */
+    get mainContentRole(): string | null;
+    /**
+     * Diagnostic tag+id+class selector for the detected main-content element, or null.
+     */
+    get mainContentSelector(): string | null;
+    /**
+     * Detected main-content element's CSS classes, or null when no main region
+     * was found. Parsed from the JSON-encoded `main_content_class_list` column.
+     */
+    get mainContentClassList(): string[] | null;
+    /**
+     * Character count of the main region's text content (denormalised
+     * aggregate written at scrape time), or null.
+     */
+    get mainContentWordCount(): number | null;
+    /**
+     * Character count of `document.body`'s text content (denormalised
+     * aggregate written at scrape time), or null.
+     */
+    get mainContentBodyWordCount(): number | null;
+    /**
+     * Number of headings within the main region (denormalised aggregate), or null.
+     */
+    get mainContentHeadingCount(): number | null;
+    /**
+     * Number of images within the main region (denormalised aggregate), or null.
+     */
+    get mainContentImageCount(): number | null;
+    /**
+     * Number of tables within the main region (denormalised aggregate), or null.
+     */
+    get mainContentTableCount(): number | null;
+    /**
+     * Number of button-like elements within the main region (denormalised
+     * aggregate), or null.
+     */
+    get mainContentButtonCount(): number | null;
+    /**
+     * Number of iframes within the main region (denormalised aggregate), or null.
+     */
+    get mainContentIframeCount(): number | null;
+    /**
+     * Number of videos within the main region (denormalised aggregate), or null.
+     */
+    get mainContentVideoCount(): number | null;
+    /**
+     * Number of audios within the main region (denormalised aggregate), or null.
+     */
+    get mainContentAudioCount(): number | null;
+    /**
+     * Number of canvases within the main region (denormalised aggregate), or null.
+     */
+    get mainContentCanvasCount(): number | null;
     /**
      * Iterable view over every flat meta column (~47 fields). Returns a frozen
      * record so consumers can pick fields by name without re-enumerating
@@ -157,6 +222,16 @@ export default class Page {
      * Use for diagnostics; specific directive flags live on `robots_*` getters.
      */
     get robots_raw(): string | null;
+    /**
+     * `document.body.scrollHeight` at the desktop-compact preset (denormalised
+     * aggregate written at scrape time), or null.
+     */
+    get scrollHeightDesktop(): number | null;
+    /**
+     * `document.body.scrollHeight` at the mobile-small preset (denormalised
+     * aggregate written at scrape time), or null.
+     */
+    get scrollHeightMobile(): number | null;
     /**
      * The reason this page was skipped during crawling, or null if it was not skipped.
      */
@@ -210,6 +285,31 @@ export default class Page {
      * @returns An array of {@link Anchor} objects representing the links on this page.
      */
     getAnchors(): Promise<Anchor[]>;
+    /**
+     * Retrieves the audios within this page's detected main content region
+     * from `page_main_content_audios`. Lazy — runs a single SELECT per call.
+     * @returns Ordered audio rows.
+     */
+    getAudios(): Promise<import("./meta/types.js").MainContentAudioRow[]>;
+    /**
+     * Retrieves the button-like elements within this page's detected main
+     * content region from `page_main_content_buttons`. Lazy — runs a single
+     * SELECT per call.
+     * @returns Ordered button rows.
+     */
+    getButtons(): Promise<import("./meta/types.js").MainContentButtonRow[]>;
+    /**
+     * Retrieves the canvases within this page's detected main content region
+     * from `page_main_content_canvases`. Lazy — runs a single SELECT per call.
+     * @returns Ordered canvas rows.
+     */
+    getCanvases(): Promise<import("./meta/types.js").MainContentCanvasRow[]>;
+    /**
+     * Retrieves the headings within this page's detected main content region
+     * from `page_main_content_headings`. Lazy — runs a single SELECT per call.
+     * @returns Ordered heading rows.
+     */
+    getHeadings(): Promise<import("./meta/types.js").MainContentHeadingRow[]>;
     /**
      * Thin wrapper that forwards this page's id to the accessor's
      * BLOB-read path. Lets callers hold a `Page` reference and ask for its
@@ -223,6 +323,12 @@ export default class Page {
      * }
      */
     getHtml(): Promise<string | null>;
+    /**
+     * Retrieves the iframes within this page's detected main content region
+     * from `page_main_content_iframes`. Lazy — runs a single SELECT per call.
+     * @returns Ordered iframe rows.
+     */
+    getIframes(): Promise<import("./meta/types.js").MainContentIframeRow[]>;
     /**
      * Retrieves the JSON-LD entries for this page from `page_jsonld`.
      * Lazy — runs a single SELECT per call. Returns entries in insertion
@@ -230,6 +336,19 @@ export default class Page {
      * @returns Ordered JSON-LD / SpeculationRules rows.
      */
     getJsonLd(): Promise<readonly JsonLdRow[]>;
+    /**
+     * Retrieves the images within this page's detected main content region
+     * from `page_main_content_images`. Lazy — runs a single SELECT per call.
+     * Distinct from the whole-page image scan (`image_items`).
+     * @returns Ordered image rows.
+     */
+    getMainContentImages(): Promise<import("./meta/types.js").MainContentImageRow[]>;
+    /**
+     * Retrieves the tables within this page's detected main content region
+     * from `page_main_content_tables`. Lazy — runs a single SELECT per call.
+     * @returns Ordered table rows.
+     */
+    getMainContentTables(): Promise<import("./meta/types.js").MainContentTableRow[]>;
     /**
      * Retrieves the referrers (incoming links) pointing to this page.
      * Uses pre-loaded data if available, otherwise queries the database.
@@ -248,6 +367,12 @@ export default class Page {
      * @returns Ordered tag rows.
      */
     getTags(): Promise<readonly TagRow[]>;
+    /**
+     * Retrieves the videos within this page's detected main content region
+     * from `page_main_content_videos`. Lazy — runs a single SELECT per call.
+     * @returns Ordered video rows.
+     */
+    getVideos(): Promise<import("./meta/types.js").MainContentVideoRow[]>;
     /**
      * Checks whether this page is an internal HTML page (not external and has `text/html` content type).
      * @returns `true` if this is an internal HTML page, `false` otherwise.
@@ -281,8 +406,26 @@ export default class Page {
         isInternalPage: boolean;
         getAnchors: Anchor[];
         getReferrers: Referrer[];
-        lang: string | number | null;
         dir: string | number | null;
+        title: string | number | null;
+        main_content_word_count: string | number | null;
+        main_content_node_name: string | number | null;
+        main_content_id: string | number | null;
+        main_content_role: string | number | null;
+        main_content_selector: string | number | null;
+        main_content_class_list: string | number | null;
+        main_content_body_word_count: string | number | null;
+        main_content_heading_count: string | number | null;
+        main_content_image_count: string | number | null;
+        main_content_table_count: string | number | null;
+        main_content_button_count: string | number | null;
+        main_content_iframe_count: string | number | null;
+        main_content_video_count: string | number | null;
+        main_content_audio_count: string | number | null;
+        main_content_canvas_count: string | number | null;
+        scroll_height_desktop: string | number | null;
+        scroll_height_mobile: string | number | null;
+        lang: string | number | null;
         charset: string | number | null;
         baseHref: string | number | null;
         viewport_raw: string | number | null;
@@ -291,7 +434,6 @@ export default class Page {
         author: string | number | null;
         generator: string | number | null;
         publisher: string | number | null;
-        title: string | number | null;
         description: string | number | null;
         keywords: string | number | null;
         robots_raw: string | number | null;

package/lib/archive/page.js

@@ -60,6 +60,23 @@ const FLAT_META_COLUMNS = [
     'tag_count',
     'jsonld_count',
     'tags_providers_csv',
+    'main_content_node_name',
+    'main_content_id',
+    'main_content_role',
+    'main_content_selector',
+    'main_content_class_list',
+    'main_content_word_count',
+    'main_content_body_word_count',
+    'main_content_heading_count',
+    'main_content_image_count',
+    'main_content_table_count',
+    'main_content_button_count',
+    'main_content_iframe_count',
+    'main_content_video_count',
+    'main_content_audio_count',
+    'main_content_canvas_count',
+    'scroll_height_desktop',
+    'scroll_height_mobile',
 ];
 /**
  * Represents a crawled page stored in the archive.
@@ -171,6 +188,104 @@ export default class Page {
     get lastCrawledAt() {
         return this.#raw.lastCrawledAt;
     }
+    /**
+     * Detected main-content element's `nodeName` (e.g. `'MAIN'`), or null when
+     * no main region was found or the page was not fully rendered.
+     */
+    get mainContentNodeName() {
+        return this.#raw.main_content_node_name;
+    }
+    /**
+     * Detected main-content element's `id`, or null.
+     */
+    get mainContentId() {
+        return this.#raw.main_content_id;
+    }
+    /**
+     * Detected main-content element's `role` attribute, or null.
+     */
+    get mainContentRole() {
+        return this.#raw.main_content_role;
+    }
+    /**
+     * Diagnostic tag+id+class selector for the detected main-content element, or null.
+     */
+    get mainContentSelector() {
+        return this.#raw.main_content_selector;
+    }
+    /**
+     * Detected main-content element's CSS classes, or null when no main region
+     * was found. Parsed from the JSON-encoded `main_content_class_list` column.
+     */
+    get mainContentClassList() {
+        if (this.#raw.main_content_class_list === null) {
+            return null;
+        }
+        return JSON.parse(this.#raw.main_content_class_list);
+    }
+    /**
+     * Character count of the main region's text content (denormalised
+     * aggregate written at scrape time), or null.
+     */
+    get mainContentWordCount() {
+        return this.#raw.main_content_word_count;
+    }
+    /**
+     * Character count of `document.body`'s text content (denormalised
+     * aggregate written at scrape time), or null.
+     */
+    get mainContentBodyWordCount() {
+        return this.#raw.main_content_body_word_count;
+    }
+    /**
+     * Number of headings within the main region (denormalised aggregate), or null.
+     */
+    get mainContentHeadingCount() {
+        return this.#raw.main_content_heading_count;
+    }
+    /**
+     * Number of images within the main region (denormalised aggregate), or null.
+     */
+    get mainContentImageCount() {
+        return this.#raw.main_content_image_count;
+    }
+    /**
+     * Number of tables within the main region (denormalised aggregate), or null.
+     */
+    get mainContentTableCount() {
+        return this.#raw.main_content_table_count;
+    }
+    /**
+     * Number of button-like elements within the main region (denormalised
+     * aggregate), or null.
+     */
+    get mainContentButtonCount() {
+        return this.#raw.main_content_button_count;
+    }
+    /**
+     * Number of iframes within the main region (denormalised aggregate), or null.
+     */
+    get mainContentIframeCount() {
+        return this.#raw.main_content_iframe_count;
+    }
+    /**
+     * Number of videos within the main region (denormalised aggregate), or null.
+     */
+    get mainContentVideoCount() {
+        return this.#raw.main_content_video_count;
+    }
+    /**
+     * Number of audios within the main region (denormalised aggregate), or null.
+     */
+    get mainContentAudioCount() {
+        return this.#raw.main_content_audio_count;
+    }
+    /**
+     * Number of canvases within the main region (denormalised aggregate), or null.
+     */
+    get mainContentCanvasCount() {
+        return this.#raw.main_content_canvas_count;
+    }
     /**
      * Iterable view over every flat meta column (~47 fields). Returns a frozen
      * record so consumers can pick fields by name without re-enumerating
@@ -280,6 +395,20 @@ export default class Page {
     get robots_raw() {
         return this.#raw.robots_raw;
     }
+    /**
+     * `document.body.scrollHeight` at the desktop-compact preset (denormalised
+     * aggregate written at scrape time), or null.
+     */
+    get scrollHeightDesktop() {
+        return this.#raw.scroll_height_desktop;
+    }
+    /**
+     * `document.body.scrollHeight` at the mobile-small preset (denormalised
+     * aggregate written at scrape time), or null.
+     */
+    get scrollHeightMobile() {
+        return this.#raw.scroll_height_mobile;
+    }
     /**
      * The reason this page was skipped during crawling, or null if it was not skipped.
      */
@@ -376,6 +505,39 @@ export default class Page {
         }
         return this.#archive.getAnchorsOnPage(this.#raw.id);
     }
+    /**
+     * Retrieves the audios within this page's detected main content region
+     * from `page_main_content_audios`. Lazy — runs a single SELECT per call.
+     * @returns Ordered audio rows.
+     */
+    async getAudios() {
+        return this.#archive.getAudiosOfPage(this.#raw.id);
+    }
+    /**
+     * Retrieves the button-like elements within this page's detected main
+     * content region from `page_main_content_buttons`. Lazy — runs a single
+     * SELECT per call.
+     * @returns Ordered button rows.
+     */
+    async getButtons() {
+        return this.#archive.getButtonsOfPage(this.#raw.id);
+    }
+    /**
+     * Retrieves the canvases within this page's detected main content region
+     * from `page_main_content_canvases`. Lazy — runs a single SELECT per call.
+     * @returns Ordered canvas rows.
+     */
+    async getCanvases() {
+        return this.#archive.getCanvasesOfPage(this.#raw.id);
+    }
+    /**
+     * Retrieves the headings within this page's detected main content region
+     * from `page_main_content_headings`. Lazy — runs a single SELECT per call.
+     * @returns Ordered heading rows.
+     */
+    async getHeadings() {
+        return this.#archive.getHeadingsOfPage(this.#raw.id);
+    }
     /**
      * Thin wrapper that forwards this page's id to the accessor's
      * BLOB-read path. Lets callers hold a `Page` reference and ask for its
@@ -391,6 +553,14 @@ export default class Page {
     async getHtml() {
         return this.#archive.getHtmlOfPage(this.#raw.id);
     }
+    /**
+     * Retrieves the iframes within this page's detected main content region
+     * from `page_main_content_iframes`. Lazy — runs a single SELECT per call.
+     * @returns Ordered iframe rows.
+     */
+    async getIframes() {
+        return this.#archive.getIframesOfPage(this.#raw.id);
+    }
     /**
      * Retrieves the JSON-LD entries for this page from `page_jsonld`.
      * Lazy — runs a single SELECT per call. Returns entries in insertion
@@ -400,6 +570,23 @@ export default class Page {
     async getJsonLd() {
         return this.#archive.getJsonLdOfPage(this.#raw.id);
     }
+    /**
+     * Retrieves the images within this page's detected main content region
+     * from `page_main_content_images`. Lazy — runs a single SELECT per call.
+     * Distinct from the whole-page image scan (`image_items`).
+     * @returns Ordered image rows.
+     */
+    async getMainContentImages() {
+        return this.#archive.getMainContentImagesOfPage(this.#raw.id);
+    }
+    /**
+     * Retrieves the tables within this page's detected main content region
+     * from `page_main_content_tables`. Lazy — runs a single SELECT per call.
+     * @returns Ordered table rows.
+     */
+    async getMainContentTables() {
+        return this.#archive.getMainContentTablesOfPage(this.#raw.id);
+    }
     /**
      * Retrieves the referrers (incoming links) pointing to this page.
      * Uses pre-loaded data if available, otherwise queries the database.
@@ -447,6 +634,14 @@ export default class Page {
     async getTags() {
         return this.#archive.getTagsOfPage(this.#raw.id);
     }
+    /**
+     * Retrieves the videos within this page's detected main content region
+     * from `page_main_content_videos`. Lazy — runs a single SELECT per call.
+     * @returns Ordered video rows.
+     */
+    async getVideos() {
+        return this.#archive.getVideosOfPage(this.#raw.id);
+    }
     /**
      * Checks whether this page is an internal HTML page (not external and has `text/html` content type).
      * @returns `true` if this is an internal HTML page, `false` otherwise.