@nitpicker/crawler 0.13.0 → 0.15.0

package/lib/archive/meta/compute-main-contents-denormalized.d.ts

@@ -0,0 +1,22 @@
+import type { MainContentsDenormalizedColumns } from './types.js';
+import type { MainContentsData, ScrollHeightData } from '@d-zero/beholder';
+/**
+ * Computes the `page_meta.main_content_*` / `scroll_height_*` denormalised
+ * aggregate columns from beholder's `MainContentsData` / `ScrollHeightData`.
+ *
+ * `mainContents` is `null` for pages that were not fully rendered (external,
+ * non-HTML, or metadata-only scrapes) — in that case every column is `null`.
+ * This mirrors {@link import('./compute-page-denormalized.js').computePageDenormalized}'s
+ * write-once-at-scrape-time pattern so list / detail reads never re-derive
+ * counts from the `page_main_content_*` child tables.
+ *
+ * Accepts `undefined` as well as `null`: `PageData` declares `mainContents`
+ * as required, but test fixtures across the codebase predate this field and
+ * omit it (`.spec.ts` files are excluded from the `tsc` build, so this goes
+ * uncaught at compile time) — `== null` tolerates both without forcing every
+ * fixture to be updated.
+ * @param mainContents - Beholder's per-page main-content metrics, or `null`/`undefined`.
+ * @param scrollHeight - Beholder's per-page scroll-height measurements, or `null`/`undefined`.
+ * @returns The seventeen denormalised columns.
+ */
+export declare function computeMainContentsDenormalized(mainContents: MainContentsData | null | undefined, scrollHeight: ScrollHeightData | null | undefined): MainContentsDenormalizedColumns;

package/lib/archive/meta/compute-main-contents-denormalized.js

@@ -0,0 +1,63 @@
+/**
+ * Computes the `page_meta.main_content_*` / `scroll_height_*` denormalised
+ * aggregate columns from beholder's `MainContentsData` / `ScrollHeightData`.
+ *
+ * `mainContents` is `null` for pages that were not fully rendered (external,
+ * non-HTML, or metadata-only scrapes) — in that case every column is `null`.
+ * This mirrors {@link import('./compute-page-denormalized.js').computePageDenormalized}'s
+ * write-once-at-scrape-time pattern so list / detail reads never re-derive
+ * counts from the `page_main_content_*` child tables.
+ *
+ * Accepts `undefined` as well as `null`: `PageData` declares `mainContents`
+ * as required, but test fixtures across the codebase predate this field and
+ * omit it (`.spec.ts` files are excluded from the `tsc` build, so this goes
+ * uncaught at compile time) — `== null` tolerates both without forcing every
+ * fixture to be updated.
+ * @param mainContents - Beholder's per-page main-content metrics, or `null`/`undefined`.
+ * @param scrollHeight - Beholder's per-page scroll-height measurements, or `null`/`undefined`.
+ * @returns The seventeen denormalised columns.
+ */
+export function computeMainContentsDenormalized(mainContents, scrollHeight) {
+    if (mainContents == null) {
+        return {
+            main_content_node_name: null,
+            main_content_id: null,
+            main_content_role: null,
+            main_content_selector: null,
+            main_content_class_list: null,
+            main_content_word_count: null,
+            main_content_body_word_count: null,
+            main_content_heading_count: null,
+            main_content_image_count: null,
+            main_content_table_count: null,
+            main_content_button_count: null,
+            main_content_iframe_count: null,
+            main_content_video_count: null,
+            main_content_audio_count: null,
+            main_content_canvas_count: null,
+            scroll_height_desktop: null,
+            scroll_height_mobile: null,
+        };
+    }
+    return {
+        main_content_node_name: mainContents.main?.nodeName ?? null,
+        main_content_id: mainContents.main?.id ?? null,
+        main_content_role: mainContents.main?.role ?? null,
+        main_content_selector: mainContents.main?.selector ?? null,
+        main_content_class_list: mainContents.main
+            ? JSON.stringify(mainContents.main.classList)
+            : null,
+        main_content_word_count: mainContents.wordCount,
+        main_content_body_word_count: mainContents.bodyWordCount,
+        main_content_heading_count: mainContents.headings.length,
+        main_content_image_count: mainContents.images.length,
+        main_content_table_count: mainContents.tables.length,
+        main_content_button_count: mainContents.buttons.length,
+        main_content_iframe_count: mainContents.iframes.length,
+        main_content_video_count: mainContents.videos.length,
+        main_content_audio_count: mainContents.audios.length,
+        main_content_canvas_count: mainContents.canvases.length,
+        scroll_height_desktop: scrollHeight?.desktop ?? null,
+        scroll_height_mobile: scrollHeight?.mobile ?? null,
+    };
+}

package/lib/archive/meta/types.d.ts

@@ -93,6 +93,269 @@ export interface PageDenormalizedColumns {
     /** Sorted unique providers, comma-separated (empty string when no tags). */
     tags_providers_csv: string;
 }
+/**
+ * Denormalised aggregates computed at write time from beholder's
+ * `MainContentsData` / `ScrollHeightData`.
+ *
+ * Stored on `page_meta` following the same pattern as
+ * {@link PageDenormalizedColumns} (`tag_count` / `jsonld_count`): the full
+ * per-element detail lives in the `page_main_content_*` child tables, while
+ * these scalar columns let list / detail reads answer "how many headings?"
+ * without joining them. All fields are `null` when the page was not fully
+ * rendered (external / non-HTML / metadata-only scrape) — see
+ * `compute-main-contents-denormalized.ts` for the `null`-in-null-out contract.
+ * @see compute-main-contents-denormalized.ts
+ * @example
+ * const denorm: MainContentsDenormalizedColumns = {
+ *   main_content_node_name: 'MAIN',
+ *   main_content_id: null,
+ *   main_content_role: null,
+ *   main_content_selector: 'main.l-main',
+ *   main_content_class_list: '["l-main"]',
+ *   main_content_word_count: 1240,
+ *   main_content_body_word_count: 1580,
+ *   main_content_heading_count: 6,
+ *   main_content_image_count: 3,
+ *   main_content_table_count: 0,
+ *   main_content_button_count: 1,
+ *   main_content_iframe_count: 0,
+ *   main_content_video_count: 0,
+ *   main_content_audio_count: 0,
+ *   main_content_canvas_count: 0,
+ *   scroll_height_desktop: 3200,
+ *   scroll_height_mobile: 5400,
+ * };
+ */
+export interface MainContentsDenormalizedColumns {
+    /** Detected main-content element's `nodeName` (e.g. `'MAIN'`), or `null`. */
+    main_content_node_name: string | null;
+    /** Detected main-content element's `id`, or `null`. */
+    main_content_id: string | null;
+    /** Detected main-content element's `role` attribute, or `null`. */
+    main_content_role: string | null;
+    /** Diagnostic tag+id+class selector for the detected element, or `null`. */
+    main_content_selector: string | null;
+    /** JSON-encoded array of the detected element's CSS classes, or `null`. */
+    main_content_class_list: string | null;
+    /** Character count of the main region's text content, or `null`. */
+    main_content_word_count: number | null;
+    /** Character count of `document.body`'s text content, or `null`. */
+    main_content_body_word_count: number | null;
+    /** Number of headings within the main region, or `null`. */
+    main_content_heading_count: number | null;
+    /** Number of images within the main region, or `null`. */
+    main_content_image_count: number | null;
+    /** Number of tables within the main region, or `null`. */
+    main_content_table_count: number | null;
+    /** Number of button-like elements within the main region, or `null`. */
+    main_content_button_count: number | null;
+    /** Number of iframes within the main region, or `null`. */
+    main_content_iframe_count: number | null;
+    /** Number of videos within the main region, or `null`. */
+    main_content_video_count: number | null;
+    /** Number of audios within the main region, or `null`. */
+    main_content_audio_count: number | null;
+    /** Number of canvases within the main region, or `null`. */
+    main_content_canvas_count: number | null;
+    /** `document.body.scrollHeight` at the desktop-compact preset, or `null`. */
+    scroll_height_desktop: number | null;
+    /** `document.body.scrollHeight` at the mobile-small preset, or `null`. */
+    scroll_height_mobile: number | null;
+}
+/**
+ * One row in the `page_main_content_headings` table.
+ * @example
+ * const row: MainContentHeadingRow = { id: 1, pageId: 42, order: 0, text: 'Welcome', level: 1 };
+ */
+export interface MainContentHeadingRow {
+    /** Auto-increment primary key. */
+    id: number;
+    /** FK to `content_items.id`. */
+    pageId: number;
+    /** 0-based DOM traversal order within the main content region. */
+    order: number;
+    /** Heading text after whitespace removal, or `null` when empty. */
+    text: string | null;
+    /** Heading level (1-6) from the tag name. */
+    level: 1 | 2 | 3 | 4 | 5 | 6;
+}
+/**
+ * One row in the `page_main_content_images` table.
+ * @example
+ * const row: MainContentImageRow = {
+ *   id: 1,
+ *   pageId: 42,
+ *   order: 0,
+ *   src: 'https://example.com/a.png',
+ *   alt: 'A photo',
+ * };
+ */
+export interface MainContentImageRow {
+    /** Auto-increment primary key. */
+    id: number;
+    /** FK to `content_items.id`. */
+    pageId: number;
+    /** 0-based DOM traversal order within the main content region. */
+    order: number;
+    /** Resolved absolute `src` URL. */
+    src: string;
+    /** `alt` attribute value (may be an empty string). */
+    alt: string;
+}
+/**
+ * One row in the `page_main_content_tables` table.
+ * @example
+ * const row: MainContentTableRow = {
+ *   id: 1,
+ *   pageId: 42,
+ *   order: 0,
+ *   rows: 3,
+ *   cols: 4,
+ *   hasHeader: 1,
+ *   hasFooter: 0,
+ *   hasMergedCell: 0,
+ * };
+ */
+export interface MainContentTableRow {
+    /** Auto-increment primary key. */
+    id: number;
+    /** FK to `content_items.id`. */
+    pageId: number;
+    /** 0-based DOM traversal order within the main content region. */
+    order: number;
+    /** Number of `<tr>` elements. */
+    rows: number;
+    /** Number of `th`/`td` cells in the first row. */
+    cols: number;
+    /** Whether the table contains a `<thead>` (raw SQLite 0/1; knex does not round-trip `.boolean()` columns back to JS `boolean` on read). */
+    hasHeader: 0 | 1;
+    /** Whether the table contains a `<tfoot>` (raw SQLite 0/1). */
+    hasFooter: 0 | 1;
+    /** Whether any cell uses `colspan` or `rowspan` (raw SQLite 0/1). */
+    hasMergedCell: 0 | 1;
+}
+/**
+ * One row in the `page_main_content_buttons` table.
+ * @example
+ * const row: MainContentButtonRow = {
+ *   id: 1,
+ *   pageId: 42,
+ *   order: 0,
+ *   nodeName: 'BUTTON',
+ *   role: null,
+ *   type: 'submit',
+ *   text: 'Send',
+ *   disabled: 0,
+ * };
+ */
+export interface MainContentButtonRow {
+    /** Auto-increment primary key. */
+    id: number;
+    /** FK to `content_items.id`. */
+    pageId: number;
+    /** 0-based DOM traversal order within the main content region. */
+    order: number;
+    /** Element tag name (e.g. `'BUTTON'`, `'A'`, `'DIV'`). */
+    nodeName: string;
+    /** `role` attribute, or `null` when absent. */
+    role: string | null;
+    /** `type` for `<button>` / `<input>`, otherwise `null`. */
+    type: string | null;
+    /** Label text after whitespace removal, or `null` when empty. */
+    text: string | null;
+    /** `true` when `disabled` or `aria-disabled="true"` (raw SQLite 0/1). */
+    disabled: 0 | 1;
+}
+/**
+ * One row in the `page_main_content_iframes` table.
+ * @example
+ * const row: MainContentIframeRow = {
+ *   id: 1,
+ *   pageId: 42,
+ *   order: 0,
+ *   src: 'https://example.com/embed',
+ *   title: null,
+ *   width: '640',
+ *   height: '360',
+ * };
+ */
+export interface MainContentIframeRow {
+    /** Auto-increment primary key. */
+    id: number;
+    /** FK to `content_items.id`. */
+    pageId: number;
+    /** 0-based DOM traversal order within the main content region. */
+    order: number;
+    /** Resolved absolute `src` URL. */
+    src: string;
+    /** `title` attribute, or `null` when absent. */
+    title: string | null;
+    /** Raw `width` attribute string, or `null` when absent. */
+    width: string | null;
+    /** Raw `height` attribute string, or `null` when absent. */
+    height: string | null;
+}
+/**
+ * One row in the `page_main_content_videos` table.
+ * @example
+ * const row: MainContentVideoRow = {
+ *   id: 1,
+ *   pageId: 42,
+ *   order: 0,
+ *   src: 'https://example.com/v.mp4',
+ *   poster: null,
+ *   width: 640,
+ *   height: 360,
+ * };
+ */
+export interface MainContentVideoRow {
+    /** Auto-increment primary key. */
+    id: number;
+    /** FK to `content_items.id`. */
+    pageId: number;
+    /** 0-based DOM traversal order within the main content region. */
+    order: number;
+    /** Resolved media URL. */
+    src: string;
+    /** Resolved `poster` URL, or `null` when unset. */
+    poster: string | null;
+    /** IDL `width` in pixels. */
+    width: number;
+    /** IDL `height` in pixels. */
+    height: number;
+}
+/**
+ * One row in the `page_main_content_audios` table.
+ * @example
+ * const row: MainContentAudioRow = { id: 1, pageId: 42, order: 0, src: 'https://example.com/a.mp3' };
+ */
+export interface MainContentAudioRow {
+    /** Auto-increment primary key. */
+    id: number;
+    /** FK to `content_items.id`. */
+    pageId: number;
+    /** 0-based DOM traversal order within the main content region. */
+    order: number;
+    /** Resolved media URL. */
+    src: string;
+}
+/**
+ * One row in the `page_main_content_canvases` table.
+ * @example
+ * const row: MainContentCanvasRow = { id: 1, pageId: 42, order: 0, width: 300, height: 150 };
+ */
+export interface MainContentCanvasRow {
+    /** Auto-increment primary key. */
+    id: number;
+    /** FK to `content_items.id`. */
+    pageId: number;
+    /** 0-based DOM traversal order within the main content region. */
+    order: number;
+    /** IDL bitmap width. */
+    width: number;
+    /** IDL bitmap height. */
+    height: number;
+}
 /**
  * One row in the `page_jsonld` table.
  *

package/lib/archive/migrate-content-items-alias-of-id.d.ts

@@ -0,0 +1,47 @@
+import type { Knex } from 'knex';
+/**
+ * Adds the `content_items.alias_of_id` column to archives created before
+ * this feature, then ensures its index exists.
+ *
+ * `content_items` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `content_items` predates this change. This
+ * mirrors {@link import('./migrate-page-meta-body-hash.js').migratePageMetaBodyHash}'s
+ * catch-up: a `hasColumn`-guarded `ALTER TABLE` for the one column
+ * `CREATE TABLE IF NOT EXISTS` cannot retrofit.
+ *
+ * Uses a raw `ALTER TABLE` (not the knex schema builder) so the retrofitted
+ * column's `REFERENCES content_items(id) DEFERRABLE INITIALLY DEFERRED`
+ * constraint matches the fresh-archive DDL bit-for-bit — the same
+ * self-referencing shape as `redirect_dest_id` (see
+ * `create-entity-tables.ts`'s JSDoc on why that FK must be deferred: a
+ * lower-id row can reference a higher-id row within the same write).
+ *
+ * The index is created here — unconditionally, after the column-add guard,
+ * not inside `createEntityTables`'s DDL — for both a fresh archive (where
+ * `alias_of_id` already exists from the DDL, so only the index still needs
+ * creating) and a legacy archive (where the column is added just above,
+ * then the index follows in the same call). `createEntityTables` runs
+ * unconditionally on every archive open, including legacy archives that
+ * still lack `alias_of_id` at that point; an unconditional
+ * `CREATE INDEX ... alias_of_id` there would fail with `no such column`
+ * before this migration ever runs. This function is the one place
+ * guaranteed to run only after the column is confirmed present, for both
+ * archive kinds.
+ *
+ * Only adds the column and its index — it does not compute values for
+ * existing rows (they stay `NULL`). That computation runs separately, from
+ * `backfillAliasOfId` during a viewer-read-model build, since it requires
+ * comparing every page's title and (for the trailing-slash tier) its
+ * `body_hash` against every other page.
+ *
+ * Idempotent: adding the column is a no-op once it exists (the index
+ * creation always runs, but `IF NOT EXISTS` makes repeat runs a no-op too).
+ * Guards on `content_items`'s existence defensively, though by the time this
+ * runs (after `initSchema`, itself after `assertCompatibleVersion` rejects
+ * pre-0.13 archives) the table is always present.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function migrateContentItemsAliasOfId(instance: Knex): Promise<void>;

package/lib/archive/migrate-content-items-alias-of-id.js

@@ -0,0 +1,58 @@
+/**
+ * Adds the `content_items.alias_of_id` column to archives created before
+ * this feature, then ensures its index exists.
+ *
+ * `content_items` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `content_items` predates this change. This
+ * mirrors {@link import('./migrate-page-meta-body-hash.js').migratePageMetaBodyHash}'s
+ * catch-up: a `hasColumn`-guarded `ALTER TABLE` for the one column
+ * `CREATE TABLE IF NOT EXISTS` cannot retrofit.
+ *
+ * Uses a raw `ALTER TABLE` (not the knex schema builder) so the retrofitted
+ * column's `REFERENCES content_items(id) DEFERRABLE INITIALLY DEFERRED`
+ * constraint matches the fresh-archive DDL bit-for-bit — the same
+ * self-referencing shape as `redirect_dest_id` (see
+ * `create-entity-tables.ts`'s JSDoc on why that FK must be deferred: a
+ * lower-id row can reference a higher-id row within the same write).
+ *
+ * The index is created here — unconditionally, after the column-add guard,
+ * not inside `createEntityTables`'s DDL — for both a fresh archive (where
+ * `alias_of_id` already exists from the DDL, so only the index still needs
+ * creating) and a legacy archive (where the column is added just above,
+ * then the index follows in the same call). `createEntityTables` runs
+ * unconditionally on every archive open, including legacy archives that
+ * still lack `alias_of_id` at that point; an unconditional
+ * `CREATE INDEX ... alias_of_id` there would fail with `no such column`
+ * before this migration ever runs. This function is the one place
+ * guaranteed to run only after the column is confirmed present, for both
+ * archive kinds.
+ *
+ * Only adds the column and its index — it does not compute values for
+ * existing rows (they stay `NULL`). That computation runs separately, from
+ * `backfillAliasOfId` during a viewer-read-model build, since it requires
+ * comparing every page's title and (for the trailing-slash tier) its
+ * `body_hash` against every other page.
+ *
+ * Idempotent: adding the column is a no-op once it exists (the index
+ * creation always runs, but `IF NOT EXISTS` makes repeat runs a no-op too).
+ * Guards on `content_items`'s existence defensively, though by the time this
+ * runs (after `initSchema`, itself after `assertCompatibleVersion` rejects
+ * pre-0.13 archives) the table is always present.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function migrateContentItemsAliasOfId(instance) {
+    const hasContentItems = await instance.schema.hasTable('content_items');
+    if (!hasContentItems) {
+        return;
+    }
+    const hasColumn = await instance.schema.hasColumn('content_items', 'alias_of_id');
+    if (!hasColumn) {
+        await instance.raw('ALTER TABLE content_items ADD COLUMN alias_of_id INTEGER REFERENCES content_items(id) DEFERRABLE INITIALLY DEFERRED');
+        // eslint-disable-next-line no-console
+        console.error('[migrate] content_items.alias_of_id column added');
+    }
+    await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_alias_of_id ON content_items(alias_of_id)');
+}

package/lib/archive/migrate-info-main-content-selector.d.ts

@@ -0,0 +1,11 @@
+import type { Knex } from 'knex';
+/**
+ * Adds the `info.mainContentSelector` column to archives created before it
+ * existed. `CREATE TABLE IF NOT EXISTS` (used for `info` itself) cannot
+ * retrofit a new column onto an already-existing table, so this lightweight,
+ * `hasColumn`-guarded `ALTER TABLE` runs on every `initSchema` call —
+ * idempotent, and self-healing for archives whose provisioning crashed
+ * partway through.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function migrateInfoMainContentSelector(instance: Knex): Promise<void>;

package/lib/archive/migrate-info-main-content-selector.js

@@ -0,0 +1,24 @@
+/**
+ * Adds the `info.mainContentSelector` column to archives created before it
+ * existed. `CREATE TABLE IF NOT EXISTS` (used for `info` itself) cannot
+ * retrofit a new column onto an already-existing table, so this lightweight,
+ * `hasColumn`-guarded `ALTER TABLE` runs on every `initSchema` call —
+ * idempotent, and self-healing for archives whose provisioning crashed
+ * partway through.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function migrateInfoMainContentSelector(instance) {
+    const hasInfo = await instance.schema.hasTable('info');
+    if (!hasInfo) {
+        return;
+    }
+    const hasColumn = await instance.schema.hasColumn('info', 'mainContentSelector');
+    if (hasColumn) {
+        return;
+    }
+    await instance.schema.table('info', (t) => {
+        t.string('mainContentSelector');
+    });
+    // eslint-disable-next-line no-console
+    console.error('[migrate] info.mainContentSelector column added');
+}

package/lib/archive/migrate-inventory-runs-invalid-skipped.d.ts

@@ -0,0 +1,11 @@
+import type { Knex } from 'knex';
+/**
+ * Adds the `inventory_runs.invalid_skipped` column to archives created
+ * before it existed. `CREATE TABLE IF NOT EXISTS` (used for `inventory_runs`
+ * itself) cannot retrofit a new column onto an already-existing table, so
+ * this lightweight, `hasColumn`-guarded `ALTER TABLE` runs on every
+ * `initSchema` call — idempotent, and self-healing for archives whose
+ * provisioning crashed partway through.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function migrateInventoryRunsInvalidSkipped(instance: Knex): Promise<void>;

package/lib/archive/migrate-inventory-runs-invalid-skipped.js

@@ -0,0 +1,24 @@
+/**
+ * Adds the `inventory_runs.invalid_skipped` column to archives created
+ * before it existed. `CREATE TABLE IF NOT EXISTS` (used for `inventory_runs`
+ * itself) cannot retrofit a new column onto an already-existing table, so
+ * this lightweight, `hasColumn`-guarded `ALTER TABLE` runs on every
+ * `initSchema` call — idempotent, and self-healing for archives whose
+ * provisioning crashed partway through.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function migrateInventoryRunsInvalidSkipped(instance) {
+    const hasTable = await instance.schema.hasTable('inventory_runs');
+    if (!hasTable) {
+        return;
+    }
+    const hasColumn = await instance.schema.hasColumn('inventory_runs', 'invalid_skipped');
+    if (hasColumn) {
+        return;
+    }
+    await instance.schema.table('inventory_runs', (t) => {
+        t.integer('invalid_skipped');
+    });
+    // eslint-disable-next-line no-console
+    console.error('[migrate] inventory_runs.invalid_skipped column added');
+}

package/lib/archive/migrate-main-contents-columns.d.ts

@@ -0,0 +1,21 @@
+import type { Knex } from 'knex';
+/**
+ * Add the beholder `MainContentsData` / `ScrollHeightData` denormalised
+ * aggregate columns to `page_meta` on archives created before this feature.
+ *
+ * `page_meta` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `page_meta` predates this change. This mirrors
+ * {@link import('./migrate-info-roots.js').migrateInfoRoots}'s `info.roots`
+ * catch-up: a `hasColumn`-guarded `ALTER TABLE` for the one column set that
+ * `CREATE TABLE IF NOT EXISTS` cannot retrofit.
+ *
+ * Idempotent: a no-op once the columns exist. Guards on `page_meta`'s
+ * existence defensively, though by the time this runs (after `initSchema`,
+ * itself after `assertCompatibleVersion` rejects pre-0.13 archives) the
+ * table is always present.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function migrateMainContentsColumns(instance: Knex): Promise<void>;

package/lib/archive/migrate-main-contents-columns.js

@@ -0,0 +1,50 @@
+/**
+ * Add the beholder `MainContentsData` / `ScrollHeightData` denormalised
+ * aggregate columns to `page_meta` on archives created before this feature.
+ *
+ * `page_meta` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `page_meta` predates this change. This mirrors
+ * {@link import('./migrate-info-roots.js').migrateInfoRoots}'s `info.roots`
+ * catch-up: a `hasColumn`-guarded `ALTER TABLE` for the one column set that
+ * `CREATE TABLE IF NOT EXISTS` cannot retrofit.
+ *
+ * Idempotent: a no-op once the columns exist. Guards on `page_meta`'s
+ * existence defensively, though by the time this runs (after `initSchema`,
+ * itself after `assertCompatibleVersion` rejects pre-0.13 archives) the
+ * table is always present.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function migrateMainContentsColumns(instance) {
+    const hasPageMeta = await instance.schema.hasTable('page_meta');
+    if (!hasPageMeta) {
+        return;
+    }
+    const hasColumn = await instance.schema.hasColumn('page_meta', 'main_content_word_count');
+    if (hasColumn) {
+        return;
+    }
+    await instance.schema.table('page_meta', (t) => {
+        t.string('main_content_node_name');
+        t.string('main_content_id');
+        t.string('main_content_role');
+        t.string('main_content_selector');
+        t.text('main_content_class_list');
+        t.integer('main_content_word_count');
+        t.integer('main_content_body_word_count');
+        t.integer('main_content_heading_count');
+        t.integer('main_content_image_count');
+        t.integer('main_content_table_count');
+        t.integer('main_content_button_count');
+        t.integer('main_content_iframe_count');
+        t.integer('main_content_video_count');
+        t.integer('main_content_audio_count');
+        t.integer('main_content_canvas_count');
+        t.integer('scroll_height_desktop');
+        t.integer('scroll_height_mobile');
+    });
+    // eslint-disable-next-line no-console
+    console.error('[migrate] page_meta.main_content_* / scroll_height_* columns added');
+}

package/lib/archive/migrate-page-meta-body-hash.d.ts

@@ -0,0 +1,39 @@
+import type { Knex } from 'knex';
+/**
+ * Adds the `page_meta.body_hash` column to archives created before this
+ * feature, then ensures its index exists.
+ *
+ * `page_meta` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `page_meta` predates this change. This mirrors
+ * {@link import('./migrate-main-contents-columns.js').migrateMainContentsColumns}'s
+ * catch-up: a `hasColumn`-guarded `ALTER TABLE` for the one column
+ * `CREATE TABLE IF NOT EXISTS` cannot retrofit.
+ *
+ * The index is created here — unconditionally, after the column-add guard,
+ * not inside `createEntityTables`'s DDL — for both a fresh archive (where
+ * `body_hash` already exists from the DDL, so only the index still needs
+ * creating) and a legacy archive (where the column is added just above,
+ * then the index follows in the same call). `createEntityTables` runs
+ * unconditionally on every archive open, including legacy archives that
+ * still lack `body_hash` at that point; an unconditional
+ * `CREATE INDEX ... body_hash` there would fail with `no such column` before
+ * this migration ever runs. This function is the one place guaranteed to
+ * run only after the column is confirmed present, for both archive kinds.
+ *
+ * Only adds the column and its index — it does not backfill values for
+ * existing rows (they stay `NULL`). That backfill runs separately, from
+ * `backfillBodyHashFromHtmlBlobs` during a viewer-read-model build, since it
+ * requires decompressing every page's stored HTML and is too heavy to run on
+ * every archive open.
+ *
+ * Idempotent: adding the column is a no-op once it exists (the index
+ * creation always runs, but `IF NOT EXISTS` makes repeat runs a no-op too).
+ * Guards on `page_meta`'s existence defensively, though by the time this
+ * runs (after `initSchema`, itself after `assertCompatibleVersion` rejects
+ * pre-0.13 archives) the table is always present.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function migratePageMetaBodyHash(instance: Knex): Promise<void>;