@nitpicker/crawler 0.13.0 → 0.14.0

package/lib/archive/database.js

@@ -8,6 +8,7 @@ import { emitError } from '../utils/error/emit-error.js';
 import { createWriteRefCaches } from './db-ops/_shared/create-write-ref-caches.js';
 import { retrySetting } from './db-ops/_shared/retry-setting.js';
 import { replaceAnalysisViolations as replaceAnalysisViolationsOp } from './db-ops/analysis/replace-analysis-violations.js';
+import { replacePageTemplates as replacePageTemplatesOp } from './db-ops/analysis/replace-page-templates.js';
 import { getAnchorsOnPage as getAnchorsOnPageOp } from './db-ops/anchors/get-anchors-on-page.js';
 import { getBaseUrl as getBaseUrlOp } from './db-ops/config/get-base-url.js';
 import { getConfig as getConfigOp } from './db-ops/config/get-config.js';
@@ -22,8 +23,16 @@ import { recordInventoryRun as recordInventoryRunOp } from './db-ops/inventory/r
 import { checkpoint as checkpointOp } from './db-ops/lifecycle/checkpoint.js';
 import { destroy as destroyOp } from './db-ops/lifecycle/destroy.js';
 import { init as initOp } from './db-ops/lifecycle/init.js';
+import { getAudiosOfPage as getAudiosOfPageOp } from './db-ops/meta/get-audios-of-page.js';
+import { getButtonsOfPage as getButtonsOfPageOp } from './db-ops/meta/get-buttons-of-page.js';
+import { getCanvasesOfPage as getCanvasesOfPageOp } from './db-ops/meta/get-canvases-of-page.js';
+import { getHeadingsOfPage as getHeadingsOfPageOp } from './db-ops/meta/get-headings-of-page.js';
+import { getIframesOfPage as getIframesOfPageOp } from './db-ops/meta/get-iframes-of-page.js';
 import { getJsonLdOfPage as getJsonLdOfPageOp } from './db-ops/meta/get-jsonld-of-page.js';
+import { getMainContentImagesOfPage as getMainContentImagesOfPageOp } from './db-ops/meta/get-main-content-images-of-page.js';
+import { getMainContentTablesOfPage as getMainContentTablesOfPageOp } from './db-ops/meta/get-main-content-tables-of-page.js';
 import { getTagsOfPage as getTagsOfPageOp } from './db-ops/meta/get-tags-of-page.js';
+import { getVideosOfPage as getVideosOfPageOp } from './db-ops/meta/get-videos-of-page.js';
 import { setUrlOrder as setUrlOrderOp } from './db-ops/pages/order/set-url-order.js';
 import { getCrawlingState as getCrawlingStateOp } from './db-ops/pages/read/get-crawling-state.js';
 import { getExistingPageUrls as getExistingPageUrlsOp } from './db-ops/pages/read/get-existing-page-urls.js';
@@ -128,6 +137,14 @@ export class Database extends EventEmitter {
     async getAnchorsOnPage(pageId) {
         return emitErrorAndRetry(this, 'Database.getAnchorsOnPage', async () => await getAnchorsOnPageOp(this.#instance, pageId), retrySetting);
     }
+    /**
+     * Retrieves all `page_main_content_audios` rows for the given page id.
+     * Delegates to {@link getAudiosOfPageOp}.
+     * @param pageId
+     */
+    async getAudiosOfPage(pageId) {
+        return emitErrorAndRetry(this, 'Database.getAudiosOfPage', async () => await getAudiosOfPageOp(this.#instance, pageId), retrySetting);
+    }
     /**
      * Retrieves the base URL of the crawl session from the `info` table.
      * Delegates to {@link getBaseUrlOp}.
@@ -137,6 +154,22 @@ export class Database extends EventEmitter {
     async getBaseUrl() {
         return emitErrorAndRetry(this, 'Database.getBaseUrl', async () => await getBaseUrlOp(this.#instance), retrySetting);
     }
+    /**
+     * Retrieves all `page_main_content_buttons` rows for the given page id.
+     * Delegates to {@link getButtonsOfPageOp}.
+     * @param pageId
+     */
+    async getButtonsOfPage(pageId) {
+        return emitErrorAndRetry(this, 'Database.getButtonsOfPage', async () => await getButtonsOfPageOp(this.#instance, pageId), retrySetting);
+    }
+    /**
+     * Retrieves all `page_main_content_canvases` rows for the given page id.
+     * Delegates to {@link getCanvasesOfPageOp}.
+     * @param pageId
+     */
+    async getCanvasesOfPage(pageId) {
+        return emitErrorAndRetry(this, 'Database.getCanvasesOfPage', async () => await getCanvasesOfPageOp(this.#instance, pageId), retrySetting);
+    }
     /**
      * Retrieves the full crawl configuration from the `info` table.
      * Delegates to {@link getConfigOp}.
@@ -174,6 +207,14 @@ export class Database extends EventEmitter {
     async getExistingResourceUrls(urls) {
         return emitError(this, 'Database.getExistingResourceUrls', async () => await getExistingResourceUrlsOp(this.#instance, urls));
     }
+    /**
+     * Retrieves all `page_main_content_headings` rows for the given page id.
+     * Delegates to {@link getHeadingsOfPageOp}.
+     * @param pageId
+     */
+    async getHeadingsOfPage(pageId) {
+        return emitErrorAndRetry(this, 'Database.getHeadingsOfPage', async () => await getHeadingsOfPageOp(this.#instance, pageId), retrySetting);
+    }
     /**
      * Reads the HTML snapshot stored as a zstd-compressed BLOB for the given page.
      * Delegates to {@link getHtmlOfPageByIdOp}.
@@ -183,6 +224,14 @@ export class Database extends EventEmitter {
     async getHtmlOfPageById(pageId) {
         return emitErrorAndRetry(this, 'Database.getHtmlOfPageById', async () => await getHtmlOfPageByIdOp(this.#instance, pageId), retrySetting);
     }
+    /**
+     * Retrieves all `page_main_content_iframes` rows for the given page id.
+     * Delegates to {@link getIframesOfPageOp}.
+     * @param pageId
+     */
+    async getIframesOfPage(pageId) {
+        return emitErrorAndRetry(this, 'Database.getIframesOfPage', async () => await getIframesOfPageOp(this.#instance, pageId), retrySetting);
+    }
     /**
      * Retrieves all `page_jsonld` rows for the given page id, parsed back into
      * {@link JsonLdRow} shape. Delegates to {@link getJsonLdOfPageOp}.
@@ -200,6 +249,22 @@ export class Database extends EventEmitter {
     getKnex() {
         return this.#instance;
     }
+    /**
+     * Retrieves all `page_main_content_images` rows for the given page id.
+     * Delegates to {@link getMainContentImagesOfPageOp}.
+     * @param pageId
+     */
+    async getMainContentImagesOfPage(pageId) {
+        return emitErrorAndRetry(this, 'Database.getMainContentImagesOfPage', async () => await getMainContentImagesOfPageOp(this.#instance, pageId), retrySetting);
+    }
+    /**
+     * Retrieves all `page_main_content_tables` rows for the given page id.
+     * Delegates to {@link getMainContentTablesOfPageOp}.
+     * @param pageId
+     */
+    async getMainContentTablesOfPage(pageId) {
+        return emitErrorAndRetry(this, 'Database.getMainContentTablesOfPage', async () => await getMainContentTablesOfPageOp(this.#instance, pageId), retrySetting);
+    }
     /**
      * Retrieves the crawl session name from the `info` table.
      * Delegates to {@link getNameOp}.
@@ -326,6 +391,14 @@ export class Database extends EventEmitter {
     async getTagsOfPage(pageId) {
         return emitErrorAndRetry(this, 'Database.getTagsOfPage', async () => await getTagsOfPageOp(this.#instance, pageId), retrySetting);
     }
+    /**
+     * Retrieves all `page_main_content_videos` rows for the given page id.
+     * Delegates to {@link getVideosOfPageOp}.
+     * @param pageId
+     */
+    async getVideosOfPage(pageId) {
+        return emitErrorAndRetry(this, 'Database.getVideosOfPage', async () => await getVideosOfPageOp(this.#instance, pageId), retrySetting);
+    }
     /**
      * Records a crawler-level (`error` channel) failure into `crawl_errors`.
      * Delegates to {@link insertCrawlErrorOp}.
@@ -421,6 +494,14 @@ export class Database extends EventEmitter {
     async replaceAnalysisViolations(violations) {
         return emitErrorAndRetry(this, 'Database.replaceAnalysisViolations', async () => await replaceAnalysisViolationsOp(this.#instance, violations), retrySetting);
     }
+    /**
+     * Replaces the stored DOM-structure template classification with a
+     * freshly generated set. Delegates to {@link replacePageTemplatesOp}.
+     * @param templateKeysByUrl - Page URL → template key.
+     */
+    async replacePageTemplates(templateKeysByUrl) {
+        return emitErrorAndRetry(this, 'Database.replacePageTemplates', async () => await replacePageTemplatesOp(this.#instance, templateKeysByUrl), retrySetting);
+    }
     /**
      * Promote previously-external pages whose URL falls under any of the new
      * scope entries back to a "needs scraping" state.

package/lib/archive/db-ops/analysis/replace-page-templates.d.ts

@@ -0,0 +1,20 @@
+import type { Knex } from 'knex';
+/**
+ * Replaces the stored DOM-structure template classification (`--templates`)
+ * with a freshly generated set.
+ *
+ * Every classified page is a full-archive, all-or-nothing recomputation
+ * (see `@nitpicker/core`'s `classifyPageTemplates`), so this always deletes
+ * every existing row before inserting the new set — there is no per-page
+ * incremental update path, matching `replaceAnalysisViolations`'s
+ * whole-table replace shape. Unlike violations, a page whose URL can't be
+ * resolved back to a `content_items` row is silently skipped rather than
+ * treated as a hard failure: losing one page's template classification
+ * (e.g. a URL-normalization mismatch between the in-memory `Page.url.href`
+ * and the stored `url_refs.url`) should not discard the rest of a
+ * potentially multi-thousand-page classification run.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param templateKeysByUrl - Page URL → template key, as produced by
+ *   `classifyPageTemplates`.
+ */
+export declare function replacePageTemplates(knex: Knex, templateKeysByUrl: ReadonlyMap<string, string>): Promise<void>;

package/lib/archive/db-ops/analysis/replace-page-templates.js

@@ -0,0 +1,52 @@
+import { eachSplitted } from '../../../utils/array/each-splitted.js';
+/**
+ * Replaces the stored DOM-structure template classification (`--templates`)
+ * with a freshly generated set.
+ *
+ * Every classified page is a full-archive, all-or-nothing recomputation
+ * (see `@nitpicker/core`'s `classifyPageTemplates`), so this always deletes
+ * every existing row before inserting the new set — there is no per-page
+ * incremental update path, matching `replaceAnalysisViolations`'s
+ * whole-table replace shape. Unlike violations, a page whose URL can't be
+ * resolved back to a `content_items` row is silently skipped rather than
+ * treated as a hard failure: losing one page's template classification
+ * (e.g. a URL-normalization mismatch between the in-memory `Page.url.href`
+ * and the stored `url_refs.url`) should not discard the rest of a
+ * potentially multi-thousand-page classification run.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param templateKeysByUrl - Page URL → template key, as produced by
+ *   `classifyPageTemplates`.
+ */
+export async function replacePageTemplates(knex, templateKeysByUrl) {
+    await knex.transaction(async (trx) => {
+        await trx('page_templates').delete();
+        if (templateKeysByUrl.size === 0) {
+            return;
+        }
+        const urls = [...templateKeysByUrl.keys()];
+        const pageIdByUrl = new Map();
+        await eachSplitted(urls, 500, async (chunk) => {
+            const pageRows = await trx('content_items')
+                .join('url_refs', 'url_refs.id', 'content_items.url_id')
+                .select('content_items.id as id', 'url_refs.url as url')
+                .whereIn('url_refs.url', chunk);
+            for (const row of pageRows) {
+                pageIdByUrl.set(row.url, row.id);
+            }
+        });
+        const rows = [];
+        for (const [url, templateKey] of templateKeysByUrl) {
+            const pageId = pageIdByUrl.get(url);
+            if (pageId == null) {
+                continue;
+            }
+            rows.push({ page_id: pageId, template_key: templateKey });
+        }
+        if (rows.length === 0) {
+            return;
+        }
+        await eachSplitted(rows, 500, async (chunk) => {
+            await trx('page_templates').insert(chunk);
+        });
+    });
+}

package/lib/archive/db-ops/config/info-column-allowlist.js

@@ -23,4 +23,5 @@ export const INFO_COLUMN_ALLOWLIST = new Set([
     'disableQueries',
     'userAgent',
     'ignoreRobots',
+    'mainContentSelector',
 ]);

package/lib/archive/db-ops/lifecycle/init.d.ts

@@ -1,17 +1,23 @@
 import type { Knex } from 'knex';
 /**
  * Initializes the database schema if tables do not exist, then runs the
- * one remaining lightweight migration (`info.roots`).
+ * remaining lightweight migrations (`info.roots`, `info.mainContentSelector`,
+ * `page_meta.main_content_*`).
  *
- * There is deliberately no per-table lazy-migration chain here:
- * `assertCompatibleVersion` (called below, before any schema work)
- * rejects every archive older than the current format, so a connection
- * that reaches `initSchema` is either brand new (initSchema provisions
- * the full schema) or was produced by `scripts/migrate-to-0.13.mjs`
- * (which guarantees the full table set before it repacks). A
- * `hasTable`-guarded catch-up migration could therefore never fire —
- * schema catch-up for old archives is the migration script's job, not
- * the open path's.
+ * There is deliberately no per-table *table-creation* migration chain here:
+ * `assertCompatibleVersion` (called below, before any schema work) rejects
+ * every archive older than the current format, so a connection that reaches
+ * `initSchema` is either brand new (`initSchema` provisions the full schema)
+ * or was produced by `scripts/migrate-to-0.13.mjs` (which guarantees the
+ * full table set before it repacks) — and `initSchema` itself re-runs
+ * `createEntityTables` / `createAdjunctTables` unconditionally on every
+ * open, self-healing any *missing table* via their internal
+ * `IF NOT EXISTS` / `hasTable` guards. What that self-healing cannot do is
+ * retrofit a *new column* onto an entity table that already exists —
+ * `CREATE TABLE IF NOT EXISTS` is a no-op once the table is present. Column
+ * additions to an existing 0.13 table are therefore the one case that still
+ * needs an explicit `hasColumn`-guarded `ALTER TABLE` here (`migrateInfoRoots`,
+ * `migrateMainContentsColumns`) rather than a DDL-string change alone.
  *
  * In read-only mode schema init + migration are SKIPPED so the same DB
  * can be opened safely by a viewer attached to a live (or interrupted)

package/lib/archive/db-ops/lifecycle/init.js

@@ -1,19 +1,27 @@
 import { applyConnectionPragmas, initSchema } from '../../init-schema.js';
 import { assertCompatibleVersion } from '../../meta/assert-compatible-version.js';
+import { migrateInfoMainContentSelector } from '../../migrate-info-main-content-selector.js';
 import { migrateInfoRoots } from '../../migrate-info-roots.js';
+import { migrateMainContentsColumns } from '../../migrate-main-contents-columns.js';
 /**
  * Initializes the database schema if tables do not exist, then runs the
- * one remaining lightweight migration (`info.roots`).
+ * remaining lightweight migrations (`info.roots`, `info.mainContentSelector`,
+ * `page_meta.main_content_*`).
  *
- * There is deliberately no per-table lazy-migration chain here:
- * `assertCompatibleVersion` (called below, before any schema work)
- * rejects every archive older than the current format, so a connection
- * that reaches `initSchema` is either brand new (initSchema provisions
- * the full schema) or was produced by `scripts/migrate-to-0.13.mjs`
- * (which guarantees the full table set before it repacks). A
- * `hasTable`-guarded catch-up migration could therefore never fire —
- * schema catch-up for old archives is the migration script's job, not
- * the open path's.
+ * There is deliberately no per-table *table-creation* migration chain here:
+ * `assertCompatibleVersion` (called below, before any schema work) rejects
+ * every archive older than the current format, so a connection that reaches
+ * `initSchema` is either brand new (`initSchema` provisions the full schema)
+ * or was produced by `scripts/migrate-to-0.13.mjs` (which guarantees the
+ * full table set before it repacks) — and `initSchema` itself re-runs
+ * `createEntityTables` / `createAdjunctTables` unconditionally on every
+ * open, self-healing any *missing table* via their internal
+ * `IF NOT EXISTS` / `hasTable` guards. What that self-healing cannot do is
+ * retrofit a *new column* onto an entity table that already exists —
+ * `CREATE TABLE IF NOT EXISTS` is a no-op once the table is present. Column
+ * additions to an existing 0.13 table are therefore the one case that still
+ * needs an explicit `hasColumn`-guarded `ALTER TABLE` here (`migrateInfoRoots`,
+ * `migrateMainContentsColumns`) rather than a DDL-string change alone.
  *
  * In read-only mode schema init + migration are SKIPPED so the same DB
  * can be opened safely by a viewer attached to a live (or interrupted)
@@ -39,4 +47,6 @@ export async function init(knex, readOnly) {
     }
     await initSchema(knex);
     await migrateInfoRoots(knex);
+    await migrateInfoMainContentSelector(knex);
+    await migrateMainContentsColumns(knex);
 }

package/lib/archive/db-ops/meta/get-audios-of-page.d.ts

@@ -0,0 +1,11 @@
+import type { MainContentAudioRow } from '../../meta/types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves all `page_main_content_audios` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertAudios`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export declare function getAudiosOfPage(knex: Knex, pageId: number): Promise<MainContentAudioRow[]>;

package/lib/archive/db-ops/meta/get-audios-of-page.js

@@ -0,0 +1,15 @@
+/**
+ * Retrieves all `page_main_content_audios` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertAudios`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export async function getAudiosOfPage(knex, pageId) {
+    return knex
+        .select('id', 'pageId', 'order', 'src')
+        .from('page_main_content_audios')
+        .where('pageId', pageId)
+        .orderBy('order', 'asc');
+}

package/lib/archive/db-ops/meta/get-buttons-of-page.d.ts

@@ -0,0 +1,11 @@
+import type { MainContentButtonRow } from '../../meta/types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves all `page_main_content_buttons` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertButtons`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export declare function getButtonsOfPage(knex: Knex, pageId: number): Promise<MainContentButtonRow[]>;

package/lib/archive/db-ops/meta/get-buttons-of-page.js

@@ -0,0 +1,15 @@
+/**
+ * Retrieves all `page_main_content_buttons` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertButtons`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export async function getButtonsOfPage(knex, pageId) {
+    return knex
+        .select('id', 'pageId', 'order', 'nodeName', 'role', 'type', 'text', 'disabled')
+        .from('page_main_content_buttons')
+        .where('pageId', pageId)
+        .orderBy('order', 'asc');
+}

package/lib/archive/db-ops/meta/get-canvases-of-page.d.ts

@@ -0,0 +1,11 @@
+import type { MainContentCanvasRow } from '../../meta/types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves all `page_main_content_canvases` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertCanvases`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export declare function getCanvasesOfPage(knex: Knex, pageId: number): Promise<MainContentCanvasRow[]>;

package/lib/archive/db-ops/meta/get-canvases-of-page.js

@@ -0,0 +1,15 @@
+/**
+ * Retrieves all `page_main_content_canvases` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertCanvases`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export async function getCanvasesOfPage(knex, pageId) {
+    return knex
+        .select('id', 'pageId', 'order', 'width', 'height')
+        .from('page_main_content_canvases')
+        .where('pageId', pageId)
+        .orderBy('order', 'asc');
+}

package/lib/archive/db-ops/meta/get-headings-of-page.d.ts

@@ -0,0 +1,11 @@
+import type { MainContentHeadingRow } from '../../meta/types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves all `page_main_content_headings` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertHeadings`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export declare function getHeadingsOfPage(knex: Knex, pageId: number): Promise<MainContentHeadingRow[]>;

package/lib/archive/db-ops/meta/get-headings-of-page.js

@@ -0,0 +1,15 @@
+/**
+ * Retrieves all `page_main_content_headings` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertHeadings`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export async function getHeadingsOfPage(knex, pageId) {
+    return knex
+        .select('id', 'pageId', 'order', 'text', 'level')
+        .from('page_main_content_headings')
+        .where('pageId', pageId)
+        .orderBy('order', 'asc');
+}

package/lib/archive/db-ops/meta/get-iframes-of-page.d.ts

@@ -0,0 +1,11 @@
+import type { MainContentIframeRow } from '../../meta/types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves all `page_main_content_iframes` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertIframes`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export declare function getIframesOfPage(knex: Knex, pageId: number): Promise<MainContentIframeRow[]>;

package/lib/archive/db-ops/meta/get-iframes-of-page.js

@@ -0,0 +1,15 @@
+/**
+ * Retrieves all `page_main_content_iframes` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertIframes`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export async function getIframesOfPage(knex, pageId) {
+    return knex
+        .select('id', 'pageId', 'order', 'src', 'title', 'width', 'height')
+        .from('page_main_content_iframes')
+        .where('pageId', pageId)
+        .orderBy('order', 'asc');
+}

package/lib/archive/db-ops/meta/get-main-content-images-of-page.d.ts

@@ -0,0 +1,11 @@
+import type { MainContentImageRow } from '../../meta/types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves all `page_main_content_images` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertMainContentImages`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export declare function getMainContentImagesOfPage(knex: Knex, pageId: number): Promise<MainContentImageRow[]>;

package/lib/archive/db-ops/meta/get-main-content-images-of-page.js

@@ -0,0 +1,15 @@
+/**
+ * Retrieves all `page_main_content_images` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertMainContentImages`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export async function getMainContentImagesOfPage(knex, pageId) {
+    return knex
+        .select('id', 'pageId', 'order', 'src', 'alt')
+        .from('page_main_content_images')
+        .where('pageId', pageId)
+        .orderBy('order', 'asc');
+}

package/lib/archive/db-ops/meta/get-main-content-tables-of-page.d.ts

@@ -0,0 +1,11 @@
+import type { MainContentTableRow } from '../../meta/types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves all `page_main_content_tables` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertMainContentTables`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export declare function getMainContentTablesOfPage(knex: Knex, pageId: number): Promise<MainContentTableRow[]>;

package/lib/archive/db-ops/meta/get-main-content-tables-of-page.js

@@ -0,0 +1,15 @@
+/**
+ * Retrieves all `page_main_content_tables` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertMainContentTables`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export async function getMainContentTablesOfPage(knex, pageId) {
+    return knex
+        .select('id', 'pageId', 'order', 'rows', 'cols', 'hasHeader', 'hasFooter', 'hasMergedCell')
+        .from('page_main_content_tables')
+        .where('pageId', pageId)
+        .orderBy('order', 'asc');
+}

package/lib/archive/db-ops/meta/get-videos-of-page.d.ts

@@ -0,0 +1,11 @@
+import type { MainContentVideoRow } from '../../meta/types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves all `page_main_content_videos` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertVideos`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export declare function getVideosOfPage(knex: Knex, pageId: number): Promise<MainContentVideoRow[]>;

package/lib/archive/db-ops/meta/get-videos-of-page.js

@@ -0,0 +1,15 @@
+/**
+ * Retrieves all `page_main_content_videos` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertVideos`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export async function getVideosOfPage(knex, pageId) {
+    return knex
+        .select('id', 'pageId', 'order', 'src', 'poster', 'width', 'height')
+        .from('page_main_content_videos')
+        .where('pageId', pageId)
+        .orderBy('order', 'asc');
+}

package/lib/archive/db-ops/pages/read/build-page-query.js

@@ -36,5 +36,5 @@ export function buildPageQuery(knex) {
         .leftJoin('url_refs as og_image_ur', 'og_image_ur.id', 'pm.og_image_url_id')
         .leftJoin('url_refs as twitter_image_ur', 'twitter_image_ur.id', 'pm.twitter_image_url_id')
         .leftJoin('json_refs as extras_ref', 'extras_ref.id', 'pm.meta_extras_json_id')
-        .select('ci.id as id', 'ur.url as url', 'ci.redirect_dest_id as redirectDestId', 'ci.scraped as scraped', 'ci.is_target as isTarget', 'ci.is_external as isExternal', 'ci.status as status', 'ci.status_text as statusText', 'ctr.raw as contentType', 'ci.content_length as contentLength', 'ci.header_set_id as headerSetId', 'pm.lang as lang', 'pm.dir as dir', 'pm.charset as charset', 'pm.base_href as baseHref', 'pm.viewport_raw as viewport_raw', 'pm.theme_color as themeColor', 'pm.application_name as applicationName', 'pm.author as author', 'pm.generator as generator', 'pm.publisher as publisher', 'title_ref.text as title', 'description_ref.text as description', 'keywords_ref.text as keywords', 'robots_raw_ref.text as robots_raw', 'pm.robots_noindex as robots_noindex', 'pm.robots_nofollow as robots_nofollow', 'pm.robots_noarchive as robots_noarchive', 'pm.robots_noimageindex as robots_noimageindex', 'pm.googlebot as googlebot', 'canonical_ur.url as canonical', 'amphtml_ur.url as amphtml', 'manifest_ur.url as manifest', 'icon_ur.url as icon_href', 'apple_ur.url as appleTouchIcon_href', 'pm.og_type as og_type', 'og_title_ref.text as og_title', 'og_url_ur.url as og_url', 'pm.og_site_name as og_site_name', 'og_description_ref.text as og_description', 'og_image_ur.url as og_image', 'pm.og_image_alt as og_image_alt', 'pm.og_image_width as og_image_width', 'pm.og_image_height as og_image_height', 'pm.og_locale as og_locale', 'pm.og_article_published_time as og_article_published_time', 'pm.og_article_modified_time as og_article_modified_time', 'pm.twitter_card as twitter_card', 'pm.twitter_site as twitter_site', 'pm.twitter_creator as twitter_creator', 'twitter_title_ref.text as twitter_title', 'twitter_description_ref.text as twitter_description', 'twitter_image_ur.url as twitter_image', 'pm.fb_app_id as fb_app_id', 'pm.verification_google as verification_google', 'pm.format_detection_telephone as formatDetection_telephone', 'ci.first_crawled_at as firstCrawledAt', 'ci.last_crawled_at as lastCrawledAt', 'pm.tag_count as tag_count', 'pm.jsonld_count as jsonld_count', 'pm.tags_providers_csv as tags_providers_csv', 'extras_ref.json_text as extras_body', 'extras_ref.codec as extras_codec', 'ci.is_skipped as isSkipped', 'ci.skip_reason as skipReason', 'ci.crawl_order as order', 'ci.source as source');
+        .select('ci.id as id', 'ur.url as url', 'ci.redirect_dest_id as redirectDestId', 'ci.scraped as scraped', 'ci.is_target as isTarget', 'ci.is_external as isExternal', 'ci.status as status', 'ci.status_text as statusText', 'ctr.raw as contentType', 'ci.content_length as contentLength', 'ci.header_set_id as headerSetId', 'pm.lang as lang', 'pm.dir as dir', 'pm.charset as charset', 'pm.base_href as baseHref', 'pm.viewport_raw as viewport_raw', 'pm.theme_color as themeColor', 'pm.application_name as applicationName', 'pm.author as author', 'pm.generator as generator', 'pm.publisher as publisher', 'title_ref.text as title', 'description_ref.text as description', 'keywords_ref.text as keywords', 'robots_raw_ref.text as robots_raw', 'pm.robots_noindex as robots_noindex', 'pm.robots_nofollow as robots_nofollow', 'pm.robots_noarchive as robots_noarchive', 'pm.robots_noimageindex as robots_noimageindex', 'pm.googlebot as googlebot', 'canonical_ur.url as canonical', 'amphtml_ur.url as amphtml', 'manifest_ur.url as manifest', 'icon_ur.url as icon_href', 'apple_ur.url as appleTouchIcon_href', 'pm.og_type as og_type', 'og_title_ref.text as og_title', 'og_url_ur.url as og_url', 'pm.og_site_name as og_site_name', 'og_description_ref.text as og_description', 'og_image_ur.url as og_image', 'pm.og_image_alt as og_image_alt', 'pm.og_image_width as og_image_width', 'pm.og_image_height as og_image_height', 'pm.og_locale as og_locale', 'pm.og_article_published_time as og_article_published_time', 'pm.og_article_modified_time as og_article_modified_time', 'pm.twitter_card as twitter_card', 'pm.twitter_site as twitter_site', 'pm.twitter_creator as twitter_creator', 'twitter_title_ref.text as twitter_title', 'twitter_description_ref.text as twitter_description', 'twitter_image_ur.url as twitter_image', 'pm.fb_app_id as fb_app_id', 'pm.verification_google as verification_google', 'pm.format_detection_telephone as formatDetection_telephone', 'ci.first_crawled_at as firstCrawledAt', 'ci.last_crawled_at as lastCrawledAt', 'pm.tag_count as tag_count', 'pm.jsonld_count as jsonld_count', 'pm.tags_providers_csv as tags_providers_csv', 'pm.main_content_node_name as main_content_node_name', 'pm.main_content_id as main_content_id', 'pm.main_content_role as main_content_role', 'pm.main_content_selector as main_content_selector', 'pm.main_content_class_list as main_content_class_list', 'pm.main_content_word_count as main_content_word_count', 'pm.main_content_body_word_count as main_content_body_word_count', 'pm.main_content_heading_count as main_content_heading_count', 'pm.main_content_image_count as main_content_image_count', 'pm.main_content_table_count as main_content_table_count', 'pm.main_content_button_count as main_content_button_count', 'pm.main_content_iframe_count as main_content_iframe_count', 'pm.main_content_video_count as main_content_video_count', 'pm.main_content_audio_count as main_content_audio_count', 'pm.main_content_canvas_count as main_content_canvas_count', 'pm.scroll_height_desktop as scroll_height_desktop', 'pm.scroll_height_mobile as scroll_height_mobile', 'extras_ref.json_text as extras_body', 'extras_ref.codec as extras_codec', 'ci.is_skipped as isSkipped', 'ci.skip_reason as skipReason', 'ci.crawl_order as order', 'ci.source as source');
 }

package/lib/archive/db-ops/pages/reset/repromote-external-pages.js

@@ -65,19 +65,20 @@ export async function repromoteExternalPages(knex, scopes, options) {
             // demotion.
         });
         // Clear the prior crawl's data for the repromoted pages. `updatePage`
-        // also replaces anchor_edges/image_items/tags/jsonld when it
-        // re-scrapes them, but only when the new scrape is non-empty — so
-        // this pre-clear is still load-bearing for pages that get
+        // also replaces anchor_edges/image_items/tags/jsonld/page_main_content_*
+        // when it re-scrapes them, but only when the new scrape is non-empty —
+        // so this pre-clear is still load-bearing for pages that get
         // repromoted but then re-scrape to nothing (or are never reached
         // again), and it is the only place `resource_ref_edges` is cleared.
         // Deleting the `page_meta` row (rather than nulling every column)
-        // clears title / description / og:* / twitter:* / meta_extras in
-        // one statement; a re-scrape re-inserts it via
-        // `ON CONFLICT(page_id) DO UPDATE`. `page_tags` / `page_jsonld` are
-        // cleared explicitly even though both tables also carry ON DELETE
-        // CASCADE — we keep the existing pattern of explicit chunked
-        // DELETEs rather than relying on CASCADE indirectly (and would not
-        // cascade anyway: the parent `content_items` row is updated, not
+        // clears title / description / og:* / twitter:* / meta_extras /
+        // main_content_* in one statement; a re-scrape re-inserts it via
+        // `ON CONFLICT(page_id) DO UPDATE`. `page_tags` / `page_jsonld` /
+        // `page_main_content_*` are cleared explicitly even though all of
+        // them also carry ON DELETE CASCADE — we keep the existing pattern
+        // of explicit chunked DELETEs rather than relying on CASCADE
+        // indirectly (and would not cascade anyway: the parent
+        // `content_items` row is updated, not
         // deleted). Orphan blobs in `page_html_blobs` are left behind; #23
         // will add GC.
         await knex('page_meta').whereIn('page_id', chunk).delete();
@@ -87,6 +88,14 @@ export async function repromoteExternalPages(knex, scopes, options) {
         await knex('page_html_ref').whereIn('page_id', chunk).delete();
         await knex('page_tags').whereIn('pageId', chunk).delete();
         await knex('page_jsonld').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_headings').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_images').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_tables').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_buttons').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_iframes').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_videos').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_audios').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_canvases').whereIn('pageId', chunk).delete();
     }
     dbLog('Repromoted %d external pages back to pending', promotedUrls.length);
     return promotedUrls;

package/lib/archive/db-ops/pages/reset/reset-failed-pages.d.ts

@@ -35,9 +35,11 @@ import type { Knex } from 'knex';
  * The page row itself is kept (id preserved) so existing
  * `anchor_edges.href_page_id` referrers stay valid, and `is_external` is
  * left untouched so the next pass re-classifies each page from the crawl
- * scope. Related `anchor_edges`, `image_items`, `resource_ref_edges`, and
- * `page_errors` rows are deleted so the re-scrape can re-insert fresh data
- * without duplicates.
+ * scope. Related `anchor_edges`, `image_items`, `resource_ref_edges`,
+ * `page_errors`, and the `page_main_content_*` child tables are deleted so
+ * the re-scrape can re-insert fresh data without duplicates — kept in sync
+ * with the `page_meta` row deletion above so a reset page's `main_content_*`
+ * counts and its child-table detail never disagree.
  *
  * SELECT and UPDATE/DELETE statements are chunked to stay below SQLite's
  * `SQLITE_LIMIT_VARIABLE_NUMBER`.