@nitpicker/crawler 0.13.0 → 0.15.0

package/lib/archive/db-ops/pages/write/insert-buttons.js

@@ -0,0 +1,29 @@
+import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+/**
+ * Replaces the page's `page_main_content_buttons` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export async function insertButtons(pageId, mainContents, trx) {
+    if (mainContents.buttons.length === 0)
+        return;
+    const rows = mainContents.buttons.map((button, order) => ({
+        pageId,
+        order,
+        nodeName: button.nodeName,
+        role: button.role,
+        type: button.type,
+        text: button.text,
+        disabled: button.disabled,
+    }));
+    await trx('page_main_content_buttons').where('pageId', pageId).delete();
+    await eachSplitted(rows, 100, async (chunk) => {
+        await trx('page_main_content_buttons').insert(chunk);
+    });
+}

package/lib/archive/db-ops/pages/write/insert-canvases.d.ts

@@ -0,0 +1,14 @@
+import type { MainContentsData } from '@d-zero/beholder';
+import type { Knex } from 'knex';
+/**
+ * Replaces the page's `page_main_content_canvases` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export declare function insertCanvases(pageId: number, mainContents: MainContentsData, trx: Knex.Transaction): Promise<void>;

package/lib/archive/db-ops/pages/write/insert-canvases.js

@@ -0,0 +1,26 @@
+import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+/**
+ * Replaces the page's `page_main_content_canvases` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export async function insertCanvases(pageId, mainContents, trx) {
+    if (mainContents.canvases.length === 0)
+        return;
+    const rows = mainContents.canvases.map((canvas, order) => ({
+        pageId,
+        order,
+        width: canvas.width,
+        height: canvas.height,
+    }));
+    await trx('page_main_content_canvases').where('pageId', pageId).delete();
+    await eachSplitted(rows, 100, async (chunk) => {
+        await trx('page_main_content_canvases').insert(chunk);
+    });
+}

package/lib/archive/db-ops/pages/write/insert-headings.d.ts

@@ -0,0 +1,15 @@
+import type { MainContentsData } from '@d-zero/beholder';
+import type { Knex } from 'knex';
+/**
+ * Replaces the page's `page_main_content_headings` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null` (non-HTML / external / metadata-only scrapes have
+ * no main-content data to write).
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export declare function insertHeadings(pageId: number, mainContents: MainContentsData, trx: Knex.Transaction): Promise<void>;

package/lib/archive/db-ops/pages/write/insert-headings.js

@@ -0,0 +1,27 @@
+import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+/**
+ * Replaces the page's `page_main_content_headings` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null` (non-HTML / external / metadata-only scrapes have
+ * no main-content data to write).
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export async function insertHeadings(pageId, mainContents, trx) {
+    if (mainContents.headings.length === 0)
+        return;
+    const rows = mainContents.headings.map((heading, order) => ({
+        pageId,
+        order,
+        text: heading.text,
+        level: heading.level,
+    }));
+    await trx('page_main_content_headings').where('pageId', pageId).delete();
+    await eachSplitted(rows, 100, async (chunk) => {
+        await trx('page_main_content_headings').insert(chunk);
+    });
+}

package/lib/archive/db-ops/pages/write/insert-iframes.d.ts

@@ -0,0 +1,14 @@
+import type { MainContentsData } from '@d-zero/beholder';
+import type { Knex } from 'knex';
+/**
+ * Replaces the page's `page_main_content_iframes` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export declare function insertIframes(pageId: number, mainContents: MainContentsData, trx: Knex.Transaction): Promise<void>;

package/lib/archive/db-ops/pages/write/insert-iframes.js

@@ -0,0 +1,28 @@
+import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+/**
+ * Replaces the page's `page_main_content_iframes` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export async function insertIframes(pageId, mainContents, trx) {
+    if (mainContents.iframes.length === 0)
+        return;
+    const rows = mainContents.iframes.map((iframe, order) => ({
+        pageId,
+        order,
+        src: iframe.src,
+        title: iframe.title,
+        width: iframe.width,
+        height: iframe.height,
+    }));
+    await trx('page_main_content_iframes').where('pageId', pageId).delete();
+    await eachSplitted(rows, 100, async (chunk) => {
+        await trx('page_main_content_iframes').insert(chunk);
+    });
+}

package/lib/archive/db-ops/pages/write/insert-main-content-images.d.ts

@@ -0,0 +1,20 @@
+import type { MainContentsData } from '@d-zero/beholder';
+import type { Knex } from 'knex';
+/**
+ * Replaces the page's `page_main_content_images` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Distinct from `image_items` (the whole-page image scan, `replaceImageItems`):
+ * this table only covers images inside the detected main-content region, so
+ * `src` is stored as a plain string rather than routed through
+ * `url_refs` / `blob_refs` — same low-cross-page-reuse rationale as the
+ * `page_meta.main_content_*` columns.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export declare function insertMainContentImages(pageId: number, mainContents: MainContentsData, trx: Knex.Transaction): Promise<void>;

package/lib/archive/db-ops/pages/write/insert-main-content-images.js

@@ -0,0 +1,32 @@
+import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+/**
+ * Replaces the page's `page_main_content_images` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Distinct from `image_items` (the whole-page image scan, `replaceImageItems`):
+ * this table only covers images inside the detected main-content region, so
+ * `src` is stored as a plain string rather than routed through
+ * `url_refs` / `blob_refs` — same low-cross-page-reuse rationale as the
+ * `page_meta.main_content_*` columns.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export async function insertMainContentImages(pageId, mainContents, trx) {
+    if (mainContents.images.length === 0)
+        return;
+    const rows = mainContents.images.map((image, order) => ({
+        pageId,
+        order,
+        src: image.src,
+        alt: image.alt,
+    }));
+    await trx('page_main_content_images').where('pageId', pageId).delete();
+    await eachSplitted(rows, 100, async (chunk) => {
+        await trx('page_main_content_images').insert(chunk);
+    });
+}

package/lib/archive/db-ops/pages/write/insert-main-content-tables.d.ts

@@ -0,0 +1,14 @@
+import type { MainContentsData } from '@d-zero/beholder';
+import type { Knex } from 'knex';
+/**
+ * Replaces the page's `page_main_content_tables` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export declare function insertMainContentTables(pageId: number, mainContents: MainContentsData, trx: Knex.Transaction): Promise<void>;

package/lib/archive/db-ops/pages/write/insert-main-content-tables.js

@@ -0,0 +1,29 @@
+import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+/**
+ * Replaces the page's `page_main_content_tables` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export async function insertMainContentTables(pageId, mainContents, trx) {
+    if (mainContents.tables.length === 0)
+        return;
+    const rows = mainContents.tables.map((table, order) => ({
+        pageId,
+        order,
+        rows: table.rows,
+        cols: table.cols,
+        hasHeader: table.hasHeader,
+        hasFooter: table.hasFooter,
+        hasMergedCell: table.hasMergedCell,
+    }));
+    await trx('page_main_content_tables').where('pageId', pageId).delete();
+    await eachSplitted(rows, 100, async (chunk) => {
+        await trx('page_main_content_tables').insert(chunk);
+    });
+}

package/lib/archive/db-ops/pages/write/insert-page.js

@@ -1,4 +1,5 @@
 import { normalizeContentType } from '../../../../crawler/normalize-content-type.js';
+import { computeMainContentsDenormalized } from '../../../meta/compute-main-contents-denormalized.js';
 import { computePageDenormalized } from '../../../meta/compute-page-denormalized.js';
 import { deriveFlatFromMeta } from '../../../meta/derive-flat-from-meta.js';
 import { deriveMetaExtras } from '../../../meta/derive-meta-extras.js';
@@ -46,6 +47,7 @@ export async function insertPage(knex, caches, page, isTarget, trx, source) {
     const pageId = await resolveContentItemId(qb, caches, page.url.withoutHashAndAuth, undefined, source);
     const flat = deriveFlatFromMeta(page.meta, page.url.href);
     const denorm = computePageDenormalized(page.meta);
+    const mainContentsDenorm = computeMainContentsDenormalized(page.mainContents, page.scrollHeight);
     const extras = deriveMetaExtras(page.meta);
     const now = Date.now();
     // Canonicalize so the stored dictionary value matches the exact-string
@@ -112,7 +114,7 @@ export async function insertPage(knex, caches, page, isTarget, trx, source) {
             entry.source = source;
         }
     }
-    await upsertPageMeta(qb, caches, pageId, flat, denorm, extras);
+    await upsertPageMeta(qb, caches, pageId, flat, denorm, page.mainContents == null ? null : mainContentsDenorm, extras);
     return pageId;
 }
 /**
@@ -125,9 +127,16 @@ export async function insertPage(knex, caches, page, isTarget, trx, source) {
  * @param pageId - The owning `content_items.id`.
  * @param flat - Flat meta columns from `deriveFlatFromMeta`.
  * @param denorm - Denormalised aggregates from `computePageDenormalized`.
+ * @param mainContentsDenorm - Denormalised `main_content_*` / `scroll_height_*`
+ *   aggregates from `computeMainContentsDenormalized`, or `null` when the
+ *   page's `mainContents` was `null` (degraded / non-HTML scrape). `null`
+ *   here omits all seventeen columns from the upsert entirely — on
+ *   `ON CONFLICT DO UPDATE` this leaves a previous full scrape's values
+ *   untouched rather than overwriting them with NULL, mirroring the
+ *   `page_main_content_*` child-table writers' same no-op-on-null guard.
  * @param extras - Nested-Meta catch-all from `deriveMetaExtras`.
  */
-async function upsertPageMeta(qb, caches, pageId, flat, denorm, extras) {
+async function upsertPageMeta(qb, caches, pageId, flat, denorm, mainContentsDenorm, extras) {
     const texts = new Set();
     for (const { source } of PAGE_META_COLUMN_MAPS.text) {
         const value = flat[source];
@@ -174,6 +183,10 @@ async function upsertPageMeta(qb, caches, pageId, flat, denorm, extras) {
         tag_count: denorm.tag_count,
         jsonld_count: denorm.jsonld_count,
         tags_providers_csv: denorm.tags_providers_csv,
+        // Omitted entirely (not set to null) when mainContentsDenorm is null —
+        // see the parameter doc above for why this preserves prior values on
+        // ON CONFLICT DO UPDATE instead of erasing them.
+        ...mainContentsDenorm,
     };
     for (const { source, target } of PAGE_META_COLUMN_MAPS.text) {
         const value = flat[source];

package/lib/archive/db-ops/pages/write/insert-videos.d.ts

@@ -0,0 +1,14 @@
+import type { MainContentsData } from '@d-zero/beholder';
+import type { Knex } from 'knex';
+/**
+ * Replaces the page's `page_main_content_videos` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export declare function insertVideos(pageId: number, mainContents: MainContentsData, trx: Knex.Transaction): Promise<void>;

package/lib/archive/db-ops/pages/write/insert-videos.js

@@ -0,0 +1,28 @@
+import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+/**
+ * Replaces the page's `page_main_content_videos` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export async function insertVideos(pageId, mainContents, trx) {
+    if (mainContents.videos.length === 0)
+        return;
+    const rows = mainContents.videos.map((video, order) => ({
+        pageId,
+        order,
+        src: video.src,
+        poster: video.poster,
+        width: video.width,
+        height: video.height,
+    }));
+    await trx('page_main_content_videos').where('pageId', pageId).delete();
+    await eachSplitted(rows, 100, async (chunk) => {
+        await trx('page_main_content_videos').insert(chunk);
+    });
+}

package/lib/archive/db-ops/pages/write/update-page.js

@@ -1,6 +1,7 @@
 import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
 import { isHtmlContentType } from '../../../../crawler/is-html-content-type.js';
 import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+import { computeBodyHash } from '../../../body-hash/compute-body-hash.js';
 import { dbLog } from '../../../debug.js';
 import { deriveLineageFromParent } from '../../../derive-lineage-from-parent.js';
 import { matchImagesToDomPaths } from '../../../populate-entity-tables/match-images-to-dom-paths.js';
@@ -9,9 +10,17 @@ import { resolveRedirectChain } from '../../../resolve-redirect-chain.js';
 import { clearWriteRefCaches } from '../../_shared/clear-write-ref-caches.js';
 import { resolveContentItemId } from '../../_shared/resolve-content-item-id.js';
 import { resolveUrlOrBlob } from '../../_shared/resolve-url-or-blob.js';
+import { insertAudios } from './insert-audios.js';
+import { insertButtons } from './insert-buttons.js';
+import { insertCanvases } from './insert-canvases.js';
+import { insertHeadings } from './insert-headings.js';
+import { insertIframes } from './insert-iframes.js';
 import { insertJsonLd } from './insert-jsonld.js';
+import { insertMainContentImages } from './insert-main-content-images.js';
+import { insertMainContentTables } from './insert-main-content-tables.js';
 import { insertPage } from './insert-page.js';
 import { insertTags } from './insert-tags.js';
+import { insertVideos } from './insert-videos.js';
 import { linkRedirectSources } from './link-redirect-sources.js';
 import { writePageHtmlBlob } from './write-page-html-blob.js';
 /**
@@ -92,6 +101,19 @@ async function updatePageInTransaction(trx, knex, caches, page, destUrlObject, s
     if (writeHtml) {
         await insertJsonLd(pageId, page.meta, trx);
     }
+    // beholder's MainContentsData is captured in the same render pass as the
+    // HTML body / JSON-LD, so it shares the same `writeHtml` + non-null gate:
+    // external / non-HTML / metadata-only scrapes never populate it.
+    if (writeHtml && page.mainContents) {
+        await insertHeadings(pageId, page.mainContents, trx);
+        await insertMainContentImages(pageId, page.mainContents, trx);
+        await insertMainContentTables(pageId, page.mainContents, trx);
+        await insertButtons(pageId, page.mainContents, trx);
+        await insertIframes(pageId, page.mainContents, trx);
+        await insertVideos(pageId, page.mainContents, trx);
+        await insertAudios(pageId, page.mainContents, trx);
+        await insertCanvases(pageId, page.mainContents, trx);
+    }
     // Chain lineage propagates FROM the originating URL
     // (`page.url`), NOT from the destination. See the matching
     // rationale in `recordRedirect`: intermediates are reached
@@ -120,6 +142,12 @@ async function updatePageInTransaction(trx, knex, caches, page, destUrlObject, s
     // content check alone expresses the intent without a redundant term.
     if (writeHtml && page.html.length > 0) {
         await writePageHtmlBlob(pageId, page.html, trx);
+        // Computed from the same `page.html` written above, in the same
+        // transaction, so `page_meta.body_hash` never observably lags the
+        // snapshot it was derived from.
+        await trx('page_meta')
+            .where('page_id', pageId)
+            .update({ body_hash: computeBodyHash(page.html) });
     }
     else if (writeHtml &&
         page.contentType !== null &&
@@ -134,7 +162,12 @@ async function updatePageInTransaction(trx, knex, caches, page, destUrlObject, s
         // stale ref can only have been written by a snapshot-capable call
         // (`setPage`); `setExternalPage` passes `writeHtml = false` and never
         // sets `html`, so it has nothing to clear.
+        //
+        // `page_meta.body_hash` is cleared alongside it for the same reason —
+        // it is derived from the same HTML `page_html_ref` points at, so it
+        // must not outlive the ref it was computed from.
         await trx('page_html_ref').where('page_id', pageId).delete();
+        await trx('page_meta').where('page_id', pageId).update({ body_hash: null });
     }
     // Re-scrape semantics: the same URL can be scraped more than once
     // (e.g. `crawl --resume`, re-visits, `--append` re-promotion). Edges

package/lib/archive/db-ops/pages/write/write-page-html-blob.js

@@ -1,5 +1,5 @@
 import { createHash } from 'node:crypto';
-import { zstdCompressSync } from 'node:zlib';
+import { compressPayload } from '../../_shared/compress-payload.js';
 /**
  * Encodes, dedups, and persists a page's HTML snapshot.
  *
@@ -20,14 +20,14 @@ import { zstdCompressSync } from 'node:zlib';
 export async function writePageHtmlBlob(pageId, html, trx) {
     const rawBytes = Buffer.from(html, 'utf8');
     const hash = createHash('sha256').update(rawBytes).digest();
-    const compressed = zstdCompressSync(rawBytes);
+    const { body, codec, sizeRaw, sizeStored } = compressPayload(rawBytes);
     await trx('page_html_blobs')
         .insert({
         hash,
-        body: compressed,
-        codec: 'zstd',
-        size_raw: rawBytes.byteLength,
-        size_stored: compressed.byteLength,
+        body,
+        codec,
+        size_raw: sizeRaw,
+        size_stored: sizeStored,
     })
         .onConflict('hash')
         .ignore();

package/lib/archive/filesystem/output-binary.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Writes raw bytes to a file at the specified path, creating parent
+ * directories if needed.
+ *
+ * Unlike {@link outputText}, the buffer is written verbatim with no UTF-8
+ * re-encoding, so callers that need byte-for-byte fidelity (e.g. archiving
+ * a source file of unknown or mixed encoding for audit purposes) are not
+ * exposed to lossy round-tripping through a JS string.
+ * @param filePath - The absolute or relative path to the file to write.
+ * @param data - The raw bytes to write.
+ */
+export declare function outputBinary(filePath: string, data: Buffer): Promise<void>;

package/lib/archive/filesystem/output-binary.js

@@ -0,0 +1,17 @@
+import { promises as fs } from 'node:fs';
+import { mkdir } from './mkdir.js';
+/**
+ * Writes raw bytes to a file at the specified path, creating parent
+ * directories if needed.
+ *
+ * Unlike {@link outputText}, the buffer is written verbatim with no UTF-8
+ * re-encoding, so callers that need byte-for-byte fidelity (e.g. archiving
+ * a source file of unknown or mixed encoding for audit purposes) are not
+ * exposed to lossy round-tripping through a JS string.
+ * @param filePath - The absolute or relative path to the file to write.
+ * @param data - The raw bytes to write.
+ */
+export async function outputBinary(filePath, data) {
+    mkdir(filePath);
+    await fs.writeFile(filePath, data);
+}

package/lib/archive/get-failed-page-messages.d.ts

@@ -1,10 +1,22 @@
 import type { Knex } from 'knex';
 /**
- * Bulk-resolve a raw error message for each given page id, using only sources
- * reachable from a {@link Knex} handle. Read order: `page_errors` (keyed by
- * `pageId`, the most direct signal a scrape attempt recorded), then
- * `crawl_errors` (keyed by `url`, the crawler-channel record for failures
- * that happened before a page row was scraped).
+ * One resolved failure message, with the timestamp it was recorded at.
+ * `createdAt` is what lets a caller (`resetFailedPages`) decide whether the
+ * failure falls inside a `network_outages` window and should be treated as
+ * retryable regardless of its classified `ErrorKind` — see
+ * `is-within-outage-window.ts`.
+ */
+export interface FailedPageMessage {
+    message: string;
+    /** Epoch ms the message was recorded (`page_errors.createdAt` or `crawl_errors.createdAt`). */
+    createdAt: number;
+}
+/**
+ * Bulk-resolve a raw error message (plus its timestamp) for each given page
+ * id, using only sources reachable from a {@link Knex} handle. Read order:
+ * `page_errors` (keyed by `pageId`, the most direct signal a scrape attempt
+ * recorded), then `crawl_errors` (keyed by `url`, the crawler-channel
+ * record for failures that happened before a page row was scraped).
  *
  * **Known limitation — pre-`crawl_errors` archives**: This helper does NOT
  * read `error.log`. The `crawl_errors` table is created empty (by
@@ -30,8 +42,8 @@ import type { Knex } from 'knex';
  * @param urls - The corresponding `pages.url` values, in the same order as
  *   `ids`. Length and indexing MUST match `ids` so the page → url join can be
  *   reconstructed without a second `pages` round-trip.
- * @returns `Map<pageId, message>` populated only for ids whose message was
- *   found in one of the consulted tables.
+ * @returns `Map<pageId, FailedPageMessage>` populated only for ids whose
+ *   message was found in one of the consulted tables.
  * @example
  * ```ts
  * const messages = await getFailedPageMessages(
@@ -41,4 +53,4 @@ import type { Knex } from 'knex';
  * );
  * ```
  */
-export declare function getFailedPageMessages(instance: Knex, ids: readonly number[], urls: readonly string[]): Promise<Map<number, string>>;
+export declare function getFailedPageMessages(instance: Knex, ids: readonly number[], urls: readonly string[]): Promise<Map<number, FailedPageMessage>>;

package/lib/archive/get-failed-page-messages.js

@@ -1,11 +1,11 @@
 /** chunk size for `WHERE … IN (?)` SQLite parameter packing. */
 const CHUNK_SIZE = 500;
 /**
- * Bulk-resolve a raw error message for each given page id, using only sources
- * reachable from a {@link Knex} handle. Read order: `page_errors` (keyed by
- * `pageId`, the most direct signal a scrape attempt recorded), then
- * `crawl_errors` (keyed by `url`, the crawler-channel record for failures
- * that happened before a page row was scraped).
+ * Bulk-resolve a raw error message (plus its timestamp) for each given page
+ * id, using only sources reachable from a {@link Knex} handle. Read order:
+ * `page_errors` (keyed by `pageId`, the most direct signal a scrape attempt
+ * recorded), then `crawl_errors` (keyed by `url`, the crawler-channel
+ * record for failures that happened before a page row was scraped).
  *
  * **Known limitation — pre-`crawl_errors` archives**: This helper does NOT
  * read `error.log`. The `crawl_errors` table is created empty (by
@@ -31,8 +31,8 @@ const CHUNK_SIZE = 500;
  * @param urls - The corresponding `pages.url` values, in the same order as
  *   `ids`. Length and indexing MUST match `ids` so the page → url join can be
  *   reconstructed without a second `pages` round-trip.
- * @returns `Map<pageId, message>` populated only for ids whose message was
- *   found in one of the consulted tables.
+ * @returns `Map<pageId, FailedPageMessage>` populated only for ids whose
+ *   message was found in one of the consulted tables.
  * @example
  * ```ts
  * const messages = await getFailedPageMessages(
@@ -61,7 +61,7 @@ export async function getFailedPageMessages(instance, ids, urls) {
             // it as `parse-error` on the next run when the rows happen to be
             // returned in a different order.
             const rows = (await instance('page_errors')
-                .select('pageId', 'message')
+                .select('pageId', 'message', 'createdAt')
                 .whereIn('pageId', chunk)
                 .orderBy('id', 'asc'));
             for (const row of rows) {
@@ -79,7 +79,10 @@ export async function getFailedPageMessages(instance, ids, urls) {
                 // `dns` / `tls` / `client-blocked` etc. — defeating
                 // `--retry-failed`'s permanent-kind exclusion.
                 if (row.message !== '' && !messageByPageId.has(row.pageId)) {
-                    messageByPageId.set(row.pageId, row.message);
+                    messageByPageId.set(row.pageId, {
+                        message: row.message,
+                        createdAt: row.createdAt,
+                    });
                 }
             }
         }
@@ -110,11 +113,22 @@ export async function getFailedPageMessages(instance, ids, urls) {
     for (let i = 0; i < missingUrls.length; i += CHUNK_SIZE) {
         const chunk = missingUrls.slice(i, i + CHUNK_SIZE);
         const rows = (await instance('crawl_errors')
-            .select('url', 'message')
+            .select('url', 'message', 'createdAt')
             .whereIn('url', chunk));
         for (const row of rows) {
-            if (row.url !== null && !urlToMessage.has(row.url)) {
-                urlToMessage.set(row.url, row.message);
+            if (row.url === null) {
+                continue;
+            }
+            // Latest-createdAt wins (fixes a previously-undefined
+            // selection among duplicate URLs — SQLite's natural scan
+            // order is implementation-defined). The most recent message
+            // is the most relevant one for both classification and outage
+            // attribution: an old NXDOMAIN followed by a network-outage
+            // blip should resolve to the outage-era message, not whichever
+            // happened to be inserted first.
+            const existing = urlToMessage.get(row.url);
+            if (existing === undefined || row.createdAt > existing.createdAt) {
+                urlToMessage.set(row.url, { message: row.message, createdAt: row.createdAt });
             }
         }
     }
@@ -123,9 +137,9 @@ export async function getFailedPageMessages(instance, ids, urls) {
         if (url === undefined) {
             continue;
         }
-        const message = urlToMessage.get(url);
-        if (message !== undefined) {
-            messageByPageId.set(id, message);
+        const resolved = urlToMessage.get(url);
+        if (resolved !== undefined) {
+            messageByPageId.set(id, resolved);
         }
     }
     return messageByPageId;

package/lib/archive/init-schema.js

@@ -14,7 +14,7 @@ import { createRefTables } from './create-ref-tables.js';
 export async function applyConnectionPragmas(instance) {
     // Foreign-key enforcement defaults to OFF on every new SQLite
     // connection. Required for ON DELETE CASCADE on `page_html_ref`,
-    // `page_tags`, and `page_jsonld` to fire.
+    // `page_tags`, `page_jsonld`, and the `page_main_content_*` tables to fire.
     await instance.raw('PRAGMA foreign_keys = ON');
     await instance.raw('PRAGMA wal_autocheckpoint = 1000');
     // Negative value = KiB of memory (64 MiB). Helps large BLOB scans.
@@ -111,6 +111,7 @@ export async function initSchema(instance) {
             t.boolean('disableQueries');
             t.string('userAgent');
             t.boolean('ignoreRobots');
+            t.string('mainContentSelector');
         });
     }
     // 0.13 ref / header dictionary tables. DDL + column-level rationale