@nitpicker/crawler 0.13.0 → 0.14.0

package/lib/archive/db-ops/pages/reset/reset-failed-pages.js

@@ -38,9 +38,11 @@ import { getFailedPageMessages } from '../../../get-failed-page-messages.js';
  * The page row itself is kept (id preserved) so existing
  * `anchor_edges.href_page_id` referrers stay valid, and `is_external` is
  * left untouched so the next pass re-classifies each page from the crawl
- * scope. Related `anchor_edges`, `image_items`, `resource_ref_edges`, and
- * `page_errors` rows are deleted so the re-scrape can re-insert fresh data
- * without duplicates.
+ * scope. Related `anchor_edges`, `image_items`, `resource_ref_edges`,
+ * `page_errors`, and the `page_main_content_*` child tables are deleted so
+ * the re-scrape can re-insert fresh data without duplicates — kept in sync
+ * with the `page_meta` row deletion above so a reset page's `main_content_*`
+ * counts and its child-table detail never disagree.
  *
  * SELECT and UPDATE/DELETE statements are chunked to stay below SQLite's
  * `SQLITE_LIMIT_VARIABLE_NUMBER`.
@@ -118,6 +120,14 @@ export async function resetFailedPages(knex) {
         await knex('page_html_ref').whereIn('page_id', chunk).delete();
         await knex('page_tags').whereIn('pageId', chunk).delete();
         await knex('page_jsonld').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_headings').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_images').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_tables').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_buttons').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_iframes').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_videos').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_audios').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_canvases').whereIn('pageId', chunk).delete();
     }
     dbLog('Reset %d failed pages back to pending', urls.length);
     return urls;

package/lib/archive/db-ops/pages/write/insert-audios.d.ts

@@ -0,0 +1,14 @@
+import type { MainContentsData } from '@d-zero/beholder';
+import type { Knex } from 'knex';
+/**
+ * Replaces the page's `page_main_content_audios` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export declare function insertAudios(pageId: number, mainContents: MainContentsData, trx: Knex.Transaction): Promise<void>;

package/lib/archive/db-ops/pages/write/insert-audios.js

@@ -0,0 +1,25 @@
+import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+/**
+ * Replaces the page's `page_main_content_audios` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export async function insertAudios(pageId, mainContents, trx) {
+    if (mainContents.audios.length === 0)
+        return;
+    const rows = mainContents.audios.map((audio, order) => ({
+        pageId,
+        order,
+        src: audio.src,
+    }));
+    await trx('page_main_content_audios').where('pageId', pageId).delete();
+    await eachSplitted(rows, 100, async (chunk) => {
+        await trx('page_main_content_audios').insert(chunk);
+    });
+}

package/lib/archive/db-ops/pages/write/insert-buttons.d.ts

@@ -0,0 +1,14 @@
+import type { MainContentsData } from '@d-zero/beholder';
+import type { Knex } from 'knex';
+/**
+ * Replaces the page's `page_main_content_buttons` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export declare function insertButtons(pageId: number, mainContents: MainContentsData, trx: Knex.Transaction): Promise<void>;

package/lib/archive/db-ops/pages/write/insert-buttons.js

@@ -0,0 +1,29 @@
+import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+/**
+ * Replaces the page's `page_main_content_buttons` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export async function insertButtons(pageId, mainContents, trx) {
+    if (mainContents.buttons.length === 0)
+        return;
+    const rows = mainContents.buttons.map((button, order) => ({
+        pageId,
+        order,
+        nodeName: button.nodeName,
+        role: button.role,
+        type: button.type,
+        text: button.text,
+        disabled: button.disabled,
+    }));
+    await trx('page_main_content_buttons').where('pageId', pageId).delete();
+    await eachSplitted(rows, 100, async (chunk) => {
+        await trx('page_main_content_buttons').insert(chunk);
+    });
+}

package/lib/archive/db-ops/pages/write/insert-canvases.d.ts

@@ -0,0 +1,14 @@
+import type { MainContentsData } from '@d-zero/beholder';
+import type { Knex } from 'knex';
+/**
+ * Replaces the page's `page_main_content_canvases` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export declare function insertCanvases(pageId: number, mainContents: MainContentsData, trx: Knex.Transaction): Promise<void>;

package/lib/archive/db-ops/pages/write/insert-canvases.js

@@ -0,0 +1,26 @@
+import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+/**
+ * Replaces the page's `page_main_content_canvases` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export async function insertCanvases(pageId, mainContents, trx) {
+    if (mainContents.canvases.length === 0)
+        return;
+    const rows = mainContents.canvases.map((canvas, order) => ({
+        pageId,
+        order,
+        width: canvas.width,
+        height: canvas.height,
+    }));
+    await trx('page_main_content_canvases').where('pageId', pageId).delete();
+    await eachSplitted(rows, 100, async (chunk) => {
+        await trx('page_main_content_canvases').insert(chunk);
+    });
+}

package/lib/archive/db-ops/pages/write/insert-headings.d.ts

@@ -0,0 +1,15 @@
+import type { MainContentsData } from '@d-zero/beholder';
+import type { Knex } from 'knex';
+/**
+ * Replaces the page's `page_main_content_headings` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null` (non-HTML / external / metadata-only scrapes have
+ * no main-content data to write).
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export declare function insertHeadings(pageId: number, mainContents: MainContentsData, trx: Knex.Transaction): Promise<void>;

package/lib/archive/db-ops/pages/write/insert-headings.js

@@ -0,0 +1,27 @@
+import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+/**
+ * Replaces the page's `page_main_content_headings` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null` (non-HTML / external / metadata-only scrapes have
+ * no main-content data to write).
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export async function insertHeadings(pageId, mainContents, trx) {
+    if (mainContents.headings.length === 0)
+        return;
+    const rows = mainContents.headings.map((heading, order) => ({
+        pageId,
+        order,
+        text: heading.text,
+        level: heading.level,
+    }));
+    await trx('page_main_content_headings').where('pageId', pageId).delete();
+    await eachSplitted(rows, 100, async (chunk) => {
+        await trx('page_main_content_headings').insert(chunk);
+    });
+}

package/lib/archive/db-ops/pages/write/insert-iframes.d.ts

@@ -0,0 +1,14 @@
+import type { MainContentsData } from '@d-zero/beholder';
+import type { Knex } from 'knex';
+/**
+ * Replaces the page's `page_main_content_iframes` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export declare function insertIframes(pageId: number, mainContents: MainContentsData, trx: Knex.Transaction): Promise<void>;

package/lib/archive/db-ops/pages/write/insert-iframes.js

@@ -0,0 +1,28 @@
+import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+/**
+ * Replaces the page's `page_main_content_iframes` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export async function insertIframes(pageId, mainContents, trx) {
+    if (mainContents.iframes.length === 0)
+        return;
+    const rows = mainContents.iframes.map((iframe, order) => ({
+        pageId,
+        order,
+        src: iframe.src,
+        title: iframe.title,
+        width: iframe.width,
+        height: iframe.height,
+    }));
+    await trx('page_main_content_iframes').where('pageId', pageId).delete();
+    await eachSplitted(rows, 100, async (chunk) => {
+        await trx('page_main_content_iframes').insert(chunk);
+    });
+}

package/lib/archive/db-ops/pages/write/insert-main-content-images.d.ts

@@ -0,0 +1,20 @@
+import type { MainContentsData } from '@d-zero/beholder';
+import type { Knex } from 'knex';
+/**
+ * Replaces the page's `page_main_content_images` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Distinct from `image_items` (the whole-page image scan, `replaceImageItems`):
+ * this table only covers images inside the detected main-content region, so
+ * `src` is stored as a plain string rather than routed through
+ * `url_refs` / `blob_refs` — same low-cross-page-reuse rationale as the
+ * `page_meta.main_content_*` columns.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export declare function insertMainContentImages(pageId: number, mainContents: MainContentsData, trx: Knex.Transaction): Promise<void>;

package/lib/archive/db-ops/pages/write/insert-main-content-images.js

@@ -0,0 +1,32 @@
+import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+/**
+ * Replaces the page's `page_main_content_images` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Distinct from `image_items` (the whole-page image scan, `replaceImageItems`):
+ * this table only covers images inside the detected main-content region, so
+ * `src` is stored as a plain string rather than routed through
+ * `url_refs` / `blob_refs` — same low-cross-page-reuse rationale as the
+ * `page_meta.main_content_*` columns.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export async function insertMainContentImages(pageId, mainContents, trx) {
+    if (mainContents.images.length === 0)
+        return;
+    const rows = mainContents.images.map((image, order) => ({
+        pageId,
+        order,
+        src: image.src,
+        alt: image.alt,
+    }));
+    await trx('page_main_content_images').where('pageId', pageId).delete();
+    await eachSplitted(rows, 100, async (chunk) => {
+        await trx('page_main_content_images').insert(chunk);
+    });
+}

package/lib/archive/db-ops/pages/write/insert-main-content-tables.d.ts

@@ -0,0 +1,14 @@
+import type { MainContentsData } from '@d-zero/beholder';
+import type { Knex } from 'knex';
+/**
+ * Replaces the page's `page_main_content_tables` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export declare function insertMainContentTables(pageId: number, mainContents: MainContentsData, trx: Knex.Transaction): Promise<void>;

package/lib/archive/db-ops/pages/write/insert-main-content-tables.js

@@ -0,0 +1,29 @@
+import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+/**
+ * Replaces the page's `page_main_content_tables` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export async function insertMainContentTables(pageId, mainContents, trx) {
+    if (mainContents.tables.length === 0)
+        return;
+    const rows = mainContents.tables.map((table, order) => ({
+        pageId,
+        order,
+        rows: table.rows,
+        cols: table.cols,
+        hasHeader: table.hasHeader,
+        hasFooter: table.hasFooter,
+        hasMergedCell: table.hasMergedCell,
+    }));
+    await trx('page_main_content_tables').where('pageId', pageId).delete();
+    await eachSplitted(rows, 100, async (chunk) => {
+        await trx('page_main_content_tables').insert(chunk);
+    });
+}

package/lib/archive/db-ops/pages/write/insert-page.js

@@ -1,4 +1,5 @@
 import { normalizeContentType } from '../../../../crawler/normalize-content-type.js';
+import { computeMainContentsDenormalized } from '../../../meta/compute-main-contents-denormalized.js';
 import { computePageDenormalized } from '../../../meta/compute-page-denormalized.js';
 import { deriveFlatFromMeta } from '../../../meta/derive-flat-from-meta.js';
 import { deriveMetaExtras } from '../../../meta/derive-meta-extras.js';
@@ -46,6 +47,7 @@ export async function insertPage(knex, caches, page, isTarget, trx, source) {
     const pageId = await resolveContentItemId(qb, caches, page.url.withoutHashAndAuth, undefined, source);
     const flat = deriveFlatFromMeta(page.meta, page.url.href);
     const denorm = computePageDenormalized(page.meta);
+    const mainContentsDenorm = computeMainContentsDenormalized(page.mainContents, page.scrollHeight);
     const extras = deriveMetaExtras(page.meta);
     const now = Date.now();
     // Canonicalize so the stored dictionary value matches the exact-string
@@ -112,7 +114,7 @@ export async function insertPage(knex, caches, page, isTarget, trx, source) {
             entry.source = source;
         }
     }
-    await upsertPageMeta(qb, caches, pageId, flat, denorm, extras);
+    await upsertPageMeta(qb, caches, pageId, flat, denorm, page.mainContents == null ? null : mainContentsDenorm, extras);
     return pageId;
 }
 /**
@@ -125,9 +127,16 @@ export async function insertPage(knex, caches, page, isTarget, trx, source) {
  * @param pageId - The owning `content_items.id`.
  * @param flat - Flat meta columns from `deriveFlatFromMeta`.
  * @param denorm - Denormalised aggregates from `computePageDenormalized`.
+ * @param mainContentsDenorm - Denormalised `main_content_*` / `scroll_height_*`
+ *   aggregates from `computeMainContentsDenormalized`, or `null` when the
+ *   page's `mainContents` was `null` (degraded / non-HTML scrape). `null`
+ *   here omits all seventeen columns from the upsert entirely — on
+ *   `ON CONFLICT DO UPDATE` this leaves a previous full scrape's values
+ *   untouched rather than overwriting them with NULL, mirroring the
+ *   `page_main_content_*` child-table writers' same no-op-on-null guard.
  * @param extras - Nested-Meta catch-all from `deriveMetaExtras`.
  */
-async function upsertPageMeta(qb, caches, pageId, flat, denorm, extras) {
+async function upsertPageMeta(qb, caches, pageId, flat, denorm, mainContentsDenorm, extras) {
     const texts = new Set();
     for (const { source } of PAGE_META_COLUMN_MAPS.text) {
         const value = flat[source];
@@ -174,6 +183,10 @@ async function upsertPageMeta(qb, caches, pageId, flat, denorm, extras) {
         tag_count: denorm.tag_count,
         jsonld_count: denorm.jsonld_count,
         tags_providers_csv: denorm.tags_providers_csv,
+        // Omitted entirely (not set to null) when mainContentsDenorm is null —
+        // see the parameter doc above for why this preserves prior values on
+        // ON CONFLICT DO UPDATE instead of erasing them.
+        ...mainContentsDenorm,
     };
     for (const { source, target } of PAGE_META_COLUMN_MAPS.text) {
         const value = flat[source];

package/lib/archive/db-ops/pages/write/insert-videos.d.ts

@@ -0,0 +1,14 @@
+import type { MainContentsData } from '@d-zero/beholder';
+import type { Knex } from 'knex';
+/**
+ * Replaces the page's `page_main_content_videos` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export declare function insertVideos(pageId: number, mainContents: MainContentsData, trx: Knex.Transaction): Promise<void>;

package/lib/archive/db-ops/pages/write/insert-videos.js

@@ -0,0 +1,28 @@
+import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+/**
+ * Replaces the page's `page_main_content_videos` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export async function insertVideos(pageId, mainContents, trx) {
+    if (mainContents.videos.length === 0)
+        return;
+    const rows = mainContents.videos.map((video, order) => ({
+        pageId,
+        order,
+        src: video.src,
+        poster: video.poster,
+        width: video.width,
+        height: video.height,
+    }));
+    await trx('page_main_content_videos').where('pageId', pageId).delete();
+    await eachSplitted(rows, 100, async (chunk) => {
+        await trx('page_main_content_videos').insert(chunk);
+    });
+}

package/lib/archive/db-ops/pages/write/update-page.js

@@ -9,9 +9,17 @@ import { resolveRedirectChain } from '../../../resolve-redirect-chain.js';
 import { clearWriteRefCaches } from '../../_shared/clear-write-ref-caches.js';
 import { resolveContentItemId } from '../../_shared/resolve-content-item-id.js';
 import { resolveUrlOrBlob } from '../../_shared/resolve-url-or-blob.js';
+import { insertAudios } from './insert-audios.js';
+import { insertButtons } from './insert-buttons.js';
+import { insertCanvases } from './insert-canvases.js';
+import { insertHeadings } from './insert-headings.js';
+import { insertIframes } from './insert-iframes.js';
 import { insertJsonLd } from './insert-jsonld.js';
+import { insertMainContentImages } from './insert-main-content-images.js';
+import { insertMainContentTables } from './insert-main-content-tables.js';
 import { insertPage } from './insert-page.js';
 import { insertTags } from './insert-tags.js';
+import { insertVideos } from './insert-videos.js';
 import { linkRedirectSources } from './link-redirect-sources.js';
 import { writePageHtmlBlob } from './write-page-html-blob.js';
 /**
@@ -92,6 +100,19 @@ async function updatePageInTransaction(trx, knex, caches, page, destUrlObject, s
     if (writeHtml) {
         await insertJsonLd(pageId, page.meta, trx);
     }
+    // beholder's MainContentsData is captured in the same render pass as the
+    // HTML body / JSON-LD, so it shares the same `writeHtml` + non-null gate:
+    // external / non-HTML / metadata-only scrapes never populate it.
+    if (writeHtml && page.mainContents) {
+        await insertHeadings(pageId, page.mainContents, trx);
+        await insertMainContentImages(pageId, page.mainContents, trx);
+        await insertMainContentTables(pageId, page.mainContents, trx);
+        await insertButtons(pageId, page.mainContents, trx);
+        await insertIframes(pageId, page.mainContents, trx);
+        await insertVideos(pageId, page.mainContents, trx);
+        await insertAudios(pageId, page.mainContents, trx);
+        await insertCanvases(pageId, page.mainContents, trx);
+    }
     // Chain lineage propagates FROM the originating URL
     // (`page.url`), NOT from the destination. See the matching
     // rationale in `recordRedirect`: intermediates are reached

package/lib/archive/init-schema.js

@@ -14,7 +14,7 @@ import { createRefTables } from './create-ref-tables.js';
 export async function applyConnectionPragmas(instance) {
     // Foreign-key enforcement defaults to OFF on every new SQLite
     // connection. Required for ON DELETE CASCADE on `page_html_ref`,
-    // `page_tags`, and `page_jsonld` to fire.
+    // `page_tags`, `page_jsonld`, and the `page_main_content_*` tables to fire.
     await instance.raw('PRAGMA foreign_keys = ON');
     await instance.raw('PRAGMA wal_autocheckpoint = 1000');
     // Negative value = KiB of memory (64 MiB). Helps large BLOB scans.
@@ -111,6 +111,7 @@ export async function initSchema(instance) {
             t.boolean('disableQueries');
             t.string('userAgent');
             t.boolean('ignoreRobots');
+            t.string('mainContentSelector');
         });
     }
     // 0.13 ref / header dictionary tables. DDL + column-level rationale

package/lib/archive/meta/compute-main-contents-denormalized.d.ts

@@ -0,0 +1,22 @@
+import type { MainContentsDenormalizedColumns } from './types.js';
+import type { MainContentsData, ScrollHeightData } from '@d-zero/beholder';
+/**
+ * Computes the `page_meta.main_content_*` / `scroll_height_*` denormalised
+ * aggregate columns from beholder's `MainContentsData` / `ScrollHeightData`.
+ *
+ * `mainContents` is `null` for pages that were not fully rendered (external,
+ * non-HTML, or metadata-only scrapes) — in that case every column is `null`.
+ * This mirrors {@link import('./compute-page-denormalized.js').computePageDenormalized}'s
+ * write-once-at-scrape-time pattern so list / detail reads never re-derive
+ * counts from the `page_main_content_*` child tables.
+ *
+ * Accepts `undefined` as well as `null`: `PageData` declares `mainContents`
+ * as required, but test fixtures across the codebase predate this field and
+ * omit it (`.spec.ts` files are excluded from the `tsc` build, so this goes
+ * uncaught at compile time) — `== null` tolerates both without forcing every
+ * fixture to be updated.
+ * @param mainContents - Beholder's per-page main-content metrics, or `null`/`undefined`.
+ * @param scrollHeight - Beholder's per-page scroll-height measurements, or `null`/`undefined`.
+ * @returns The seventeen denormalised columns.
+ */
+export declare function computeMainContentsDenormalized(mainContents: MainContentsData | null | undefined, scrollHeight: ScrollHeightData | null | undefined): MainContentsDenormalizedColumns;

package/lib/archive/meta/compute-main-contents-denormalized.js

@@ -0,0 +1,63 @@
+/**
+ * Computes the `page_meta.main_content_*` / `scroll_height_*` denormalised
+ * aggregate columns from beholder's `MainContentsData` / `ScrollHeightData`.
+ *
+ * `mainContents` is `null` for pages that were not fully rendered (external,
+ * non-HTML, or metadata-only scrapes) — in that case every column is `null`.
+ * This mirrors {@link import('./compute-page-denormalized.js').computePageDenormalized}'s
+ * write-once-at-scrape-time pattern so list / detail reads never re-derive
+ * counts from the `page_main_content_*` child tables.
+ *
+ * Accepts `undefined` as well as `null`: `PageData` declares `mainContents`
+ * as required, but test fixtures across the codebase predate this field and
+ * omit it (`.spec.ts` files are excluded from the `tsc` build, so this goes
+ * uncaught at compile time) — `== null` tolerates both without forcing every
+ * fixture to be updated.
+ * @param mainContents - Beholder's per-page main-content metrics, or `null`/`undefined`.
+ * @param scrollHeight - Beholder's per-page scroll-height measurements, or `null`/`undefined`.
+ * @returns The seventeen denormalised columns.
+ */
+export function computeMainContentsDenormalized(mainContents, scrollHeight) {
+    if (mainContents == null) {
+        return {
+            main_content_node_name: null,
+            main_content_id: null,
+            main_content_role: null,
+            main_content_selector: null,
+            main_content_class_list: null,
+            main_content_word_count: null,
+            main_content_body_word_count: null,
+            main_content_heading_count: null,
+            main_content_image_count: null,
+            main_content_table_count: null,
+            main_content_button_count: null,
+            main_content_iframe_count: null,
+            main_content_video_count: null,
+            main_content_audio_count: null,
+            main_content_canvas_count: null,
+            scroll_height_desktop: null,
+            scroll_height_mobile: null,
+        };
+    }
+    return {
+        main_content_node_name: mainContents.main?.nodeName ?? null,
+        main_content_id: mainContents.main?.id ?? null,
+        main_content_role: mainContents.main?.role ?? null,
+        main_content_selector: mainContents.main?.selector ?? null,
+        main_content_class_list: mainContents.main
+            ? JSON.stringify(mainContents.main.classList)
+            : null,
+        main_content_word_count: mainContents.wordCount,
+        main_content_body_word_count: mainContents.bodyWordCount,
+        main_content_heading_count: mainContents.headings.length,
+        main_content_image_count: mainContents.images.length,
+        main_content_table_count: mainContents.tables.length,
+        main_content_button_count: mainContents.buttons.length,
+        main_content_iframe_count: mainContents.iframes.length,
+        main_content_video_count: mainContents.videos.length,
+        main_content_audio_count: mainContents.audios.length,
+        main_content_canvas_count: mainContents.canvases.length,
+        scroll_height_desktop: scrollHeight?.desktop ?? null,
+        scroll_height_mobile: scrollHeight?.mobile ?? null,
+    };
+}