@nitpicker/crawler 0.13.0 → 0.15.0

package/lib/archive/db-ops/meta/get-main-content-images-of-page.js

@@ -0,0 +1,15 @@
+/**
+ * Retrieves all `page_main_content_images` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertMainContentImages`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export async function getMainContentImagesOfPage(knex, pageId) {
+    return knex
+        .select('id', 'pageId', 'order', 'src', 'alt')
+        .from('page_main_content_images')
+        .where('pageId', pageId)
+        .orderBy('order', 'asc');
+}

package/lib/archive/db-ops/meta/get-main-content-tables-of-page.d.ts

@@ -0,0 +1,11 @@
+import type { MainContentTableRow } from '../../meta/types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves all `page_main_content_tables` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertMainContentTables`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export declare function getMainContentTablesOfPage(knex: Knex, pageId: number): Promise<MainContentTableRow[]>;

package/lib/archive/db-ops/meta/get-main-content-tables-of-page.js

@@ -0,0 +1,15 @@
+/**
+ * Retrieves all `page_main_content_tables` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertMainContentTables`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export async function getMainContentTablesOfPage(knex, pageId) {
+    return knex
+        .select('id', 'pageId', 'order', 'rows', 'cols', 'hasHeader', 'hasFooter', 'hasMergedCell')
+        .from('page_main_content_tables')
+        .where('pageId', pageId)
+        .orderBy('order', 'asc');
+}

package/lib/archive/db-ops/meta/get-videos-of-page.d.ts

@@ -0,0 +1,11 @@
+import type { MainContentVideoRow } from '../../meta/types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves all `page_main_content_videos` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertVideos`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export declare function getVideosOfPage(knex: Knex, pageId: number): Promise<MainContentVideoRow[]>;

package/lib/archive/db-ops/meta/get-videos-of-page.js

@@ -0,0 +1,15 @@
+/**
+ * Retrieves all `page_main_content_videos` rows for the given page id, in
+ * DOM traversal order.
+ *
+ * Read-side counterpart to `insertVideos`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export async function getVideosOfPage(knex, pageId) {
+    return knex
+        .select('id', 'pageId', 'order', 'src', 'poster', 'width', 'height')
+        .from('page_main_content_videos')
+        .where('pageId', pageId)
+        .orderBy('order', 'asc');
+}

package/lib/archive/db-ops/outages/close-network-outage.d.ts

@@ -0,0 +1,12 @@
+import type { Knex } from 'knex';
+/**
+ * Close an outage row by stamping `ended_at` — but ONLY if it is still
+ * open. The `whereNull('ended_at')` guard is what makes this idempotent: a
+ * second call (e.g. a duplicate recovery-probe success racing the first)
+ * matches zero rows and is a silent no-op, rather than overwriting an
+ * already-recorded `ended_at` with a later timestamp.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param id - The `network_outages.id` to close.
+ * @param endedAt - Epoch ms the outage is considered to have ended.
+ */
+export declare function closeNetworkOutage(knex: Knex, id: number, endedAt: number): Promise<void>;

package/lib/archive/db-ops/outages/close-network-outage.js

@@ -0,0 +1,15 @@
+/**
+ * Close an outage row by stamping `ended_at` — but ONLY if it is still
+ * open. The `whereNull('ended_at')` guard is what makes this idempotent: a
+ * second call (e.g. a duplicate recovery-probe success racing the first)
+ * matches zero rows and is a silent no-op, rather than overwriting an
+ * already-recorded `ended_at` with a later timestamp.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param id - The `network_outages.id` to close.
+ * @param endedAt - Epoch ms the outage is considered to have ended.
+ */
+export async function closeNetworkOutage(knex, id, endedAt) {
+    await knex('network_outages').where({ id }).whereNull('ended_at').update({
+        ended_at: endedAt,
+    });
+}

package/lib/archive/db-ops/outages/close-stale-open-network-outages.d.ts

@@ -0,0 +1,26 @@
+import type { Knex } from 'knex';
+/**
+ * Durably close every `network_outages` row still `ended_at = NULL` at the
+ * start of a writer session.
+ *
+ * A row is left open only when the crawl process was killed (Ctrl-C / OOM
+ * / SIGKILL) mid-outage, before a recovery probe could close it. Called
+ * once from `db-ops/lifecycle/init.ts` — right after `initSchema` (which
+ * guarantees the table exists) and before anything else touches
+ * `network_outages` — so that by the time `resetFailedPages` /
+ * `listDnsBurnedHostCandidates` / any other reader runs, no row can still
+ * be open from a PRIOR session. (A row opened by THIS session cannot exist
+ * yet at this point in the boot sequence — the sliding-window detector
+ * only starts once the crawl loop begins.)
+ *
+ * This complements, but does not replace, `list-network-outages.ts`'s
+ * on-the-fly clamp: that read-side resolution stays in place as a
+ * defensive fallback, but after this runs there should be nothing left for
+ * it to resolve.
+ *
+ * Idempotent: a row closed by a previous call (or by a normal
+ * recovery-probe success) is simply absent from the `whereNull('ended_at')`
+ * scan, so re-running this on every writer open is always safe.
+ * @param knex - Knex query builder connected to the archive DB.
+ */
+export declare function closeStaleOpenNetworkOutages(knex: Knex): Promise<void>;

package/lib/archive/db-ops/outages/close-stale-open-network-outages.js

@@ -0,0 +1,42 @@
+import { closeNetworkOutage } from './close-network-outage.js';
+import { computeOutageClampTimestamp } from './compute-outage-clamp-timestamp.js';
+/**
+ * Durably close every `network_outages` row still `ended_at = NULL` at the
+ * start of a writer session.
+ *
+ * A row is left open only when the crawl process was killed (Ctrl-C / OOM
+ * / SIGKILL) mid-outage, before a recovery probe could close it. Called
+ * once from `db-ops/lifecycle/init.ts` — right after `initSchema` (which
+ * guarantees the table exists) and before anything else touches
+ * `network_outages` — so that by the time `resetFailedPages` /
+ * `listDnsBurnedHostCandidates` / any other reader runs, no row can still
+ * be open from a PRIOR session. (A row opened by THIS session cannot exist
+ * yet at this point in the boot sequence — the sliding-window detector
+ * only starts once the crawl loop begins.)
+ *
+ * This complements, but does not replace, `list-network-outages.ts`'s
+ * on-the-fly clamp: that read-side resolution stays in place as a
+ * defensive fallback, but after this runs there should be nothing left for
+ * it to resolve.
+ *
+ * Idempotent: a row closed by a previous call (or by a normal
+ * recovery-probe success) is simply absent from the `whereNull('ended_at')`
+ * scan, so re-running this on every writer open is always safe.
+ * @param knex - Knex query builder connected to the archive DB.
+ */
+export async function closeStaleOpenNetworkOutages(knex) {
+    const hasTable = await knex.schema.hasTable('network_outages');
+    if (!hasTable) {
+        return;
+    }
+    const openRows = (await knex('network_outages')
+        .whereNull('ended_at')
+        .select('id', 'started_at'));
+    if (openRows.length === 0) {
+        return;
+    }
+    const clamp = await computeOutageClampTimestamp(knex);
+    for (const row of openRows) {
+        await closeNetworkOutage(knex, row.id, Math.max(clamp, row.started_at));
+    }
+}

package/lib/archive/db-ops/outages/compute-outage-clamp-timestamp.d.ts

@@ -0,0 +1,26 @@
+import type { Knex } from 'knex';
+/**
+ * Compute the timestamp used to resolve an outage row whose `ended_at` is
+ * still `NULL` (the crawl process was killed before a recovery probe could
+ * close it) into a concrete, bounded window.
+ *
+ * The archive's own most-recent observations — the latest `crawl_errors`
+ * timestamp and the latest `content_items.last_crawled_at` — are the only
+ * evidence available for "when did activity in this archive last happen",
+ * so the later of the two stands in for "the outage cannot have lasted
+ * past this point, because the archive shows activity here". Using the
+ * LARGER of the two (not just `crawl_errors`) matters because a session
+ * that crashed mid-outage may have its last successful page write be more
+ * recent than its last recorded error, or vice versa.
+ *
+ * Both source tables are guaranteed to exist by the time this runs — this
+ * helper is only ever called from writer-context code (see
+ * `list-network-outages.ts` / the boot-time stale-open finalizer), and
+ * `initSchema` creates both `crawl_errors` and `content_items` before any
+ * writer session's first query.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The larger of `MAX(crawl_errors.createdAt)` and
+ *   `MAX(content_items.last_crawled_at)`, or `0` if the archive has neither
+ *   (a brand-new archive with no activity yet).
+ */
+export declare function computeOutageClampTimestamp(knex: Knex): Promise<number>;

package/lib/archive/db-ops/outages/compute-outage-clamp-timestamp.js

@@ -0,0 +1,31 @@
+/**
+ * Compute the timestamp used to resolve an outage row whose `ended_at` is
+ * still `NULL` (the crawl process was killed before a recovery probe could
+ * close it) into a concrete, bounded window.
+ *
+ * The archive's own most-recent observations — the latest `crawl_errors`
+ * timestamp and the latest `content_items.last_crawled_at` — are the only
+ * evidence available for "when did activity in this archive last happen",
+ * so the later of the two stands in for "the outage cannot have lasted
+ * past this point, because the archive shows activity here". Using the
+ * LARGER of the two (not just `crawl_errors`) matters because a session
+ * that crashed mid-outage may have its last successful page write be more
+ * recent than its last recorded error, or vice versa.
+ *
+ * Both source tables are guaranteed to exist by the time this runs — this
+ * helper is only ever called from writer-context code (see
+ * `list-network-outages.ts` / the boot-time stale-open finalizer), and
+ * `initSchema` creates both `crawl_errors` and `content_items` before any
+ * writer session's first query.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The larger of `MAX(crawl_errors.createdAt)` and
+ *   `MAX(content_items.last_crawled_at)`, or `0` if the archive has neither
+ *   (a brand-new archive with no activity yet).
+ */
+export async function computeOutageClampTimestamp(knex) {
+    const [crawlErrorsRow] = (await knex('crawl_errors').max('createdAt as maxCreatedAt'));
+    const [contentItemsRow] = (await knex('content_items').max('last_crawled_at as maxLastCrawledAt'));
+    const latestError = crawlErrorsRow?.maxCreatedAt ?? 0;
+    const latestCrawl = contentItemsRow?.maxLastCrawledAt ?? 0;
+    return Math.max(latestError, latestCrawl);
+}

package/lib/archive/db-ops/outages/insert-network-outage.d.ts

@@ -0,0 +1,14 @@
+import type { InsertNetworkOutageParams } from '../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Appends one row to the `network_outages` journal, with `ended_at` left
+ * `NULL` — the row starts life as an open outage.
+ *
+ * Called the moment a recovery probe CONFIRMS a suspect outage (i.e. the
+ * probe itself failed, not merely the sliding-window threshold trip). See
+ * `close-network-outage.ts` for how the row is later closed.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param params - The confirmed-outage fields to record.
+ * @returns The autoincremented `id` of the newly-inserted row.
+ */
+export declare function insertNetworkOutage(knex: Knex, params: InsertNetworkOutageParams): Promise<number>;

package/lib/archive/db-ops/outages/insert-network-outage.js

@@ -0,0 +1,29 @@
+/**
+ * Appends one row to the `network_outages` journal, with `ended_at` left
+ * `NULL` — the row starts life as an open outage.
+ *
+ * Called the moment a recovery probe CONFIRMS a suspect outage (i.e. the
+ * probe itself failed, not merely the sliding-window threshold trip). See
+ * `close-network-outage.ts` for how the row is later closed.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param params - The confirmed-outage fields to record.
+ * @returns The autoincremented `id` of the newly-inserted row.
+ */
+export async function insertNetworkOutage(knex, params) {
+    const inserted = await knex
+        .from('network_outages')
+        .insert({
+        started_at: params.startedAt,
+        detected_at: params.detectedAt,
+        ended_at: null,
+        probe_host: params.probeHost,
+        trigger_error_count: params.triggerErrorCount,
+        trigger_host_count: params.triggerHostCount,
+    })
+        .returning('id');
+    const id = inserted[0]?.id;
+    if (typeof id !== 'number') {
+        throw new TypeError('insertNetworkOutage: INSERT returned no row id');
+    }
+    return id;
+}

package/lib/archive/db-ops/outages/list-network-outages.d.ts

@@ -0,0 +1,28 @@
+import type { OutageWindow } from '../../../is-within-outage-window.js';
+import type { Knex } from 'knex';
+/**
+ * List every recorded outage as a resolved (closed) {@link OutageWindow},
+ * suitable for `isWithinOutageWindow`.
+ *
+ * This is the crawler-internal counterpart to `@nitpicker/query`'s richer
+ * `listNetworkOutages` (full row shape, for CLI/MCP display) — this one
+ * exists purely to feed the three write-path consumers
+ * (`resetFailedPages`, `listDnsBurnedHostCandidates`, the gate itself) that
+ * only need "was this timestamp inside an outage", never the row's other
+ * columns.
+ *
+ * Any row whose `ended_at` is still `NULL` (the crawl process was killed
+ * before recovery) is resolved on the fly via
+ * {@link computeOutageClampTimestamp} rather than ever being returned as an
+ * unbounded window — see `is-within-outage-window.ts`'s `OutageWindow`
+ * docstring for why an open-ended window would be a correctness bug (every
+ * later error would retroactively read as network-caused, forever). This
+ * on-the-fly resolution does NOT persist to the row — it is a defensive
+ * fallback independent of whichever boot-time finalizer durably closes
+ * stale-open rows.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns Resolved outage windows. Empty when the archive predates the
+ *   `network_outages` table (self-healed on next writer open, so this is
+ *   never a permanent state) or has recorded no outages.
+ */
+export declare function listNetworkOutages(knex: Knex): Promise<OutageWindow[]>;

package/lib/archive/db-ops/outages/list-network-outages.js

@@ -0,0 +1,42 @@
+import { computeOutageClampTimestamp } from './compute-outage-clamp-timestamp.js';
+/**
+ * List every recorded outage as a resolved (closed) {@link OutageWindow},
+ * suitable for `isWithinOutageWindow`.
+ *
+ * This is the crawler-internal counterpart to `@nitpicker/query`'s richer
+ * `listNetworkOutages` (full row shape, for CLI/MCP display) — this one
+ * exists purely to feed the three write-path consumers
+ * (`resetFailedPages`, `listDnsBurnedHostCandidates`, the gate itself) that
+ * only need "was this timestamp inside an outage", never the row's other
+ * columns.
+ *
+ * Any row whose `ended_at` is still `NULL` (the crawl process was killed
+ * before recovery) is resolved on the fly via
+ * {@link computeOutageClampTimestamp} rather than ever being returned as an
+ * unbounded window — see `is-within-outage-window.ts`'s `OutageWindow`
+ * docstring for why an open-ended window would be a correctness bug (every
+ * later error would retroactively read as network-caused, forever). This
+ * on-the-fly resolution does NOT persist to the row — it is a defensive
+ * fallback independent of whichever boot-time finalizer durably closes
+ * stale-open rows.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns Resolved outage windows. Empty when the archive predates the
+ *   `network_outages` table (self-healed on next writer open, so this is
+ *   never a permanent state) or has recorded no outages.
+ */
+export async function listNetworkOutages(knex) {
+    const hasTable = await knex.schema.hasTable('network_outages');
+    if (!hasTable) {
+        return [];
+    }
+    const rows = (await knex('network_outages').select('started_at', 'ended_at'));
+    if (rows.length === 0) {
+        return [];
+    }
+    const hasOpenRow = rows.some((row) => row.ended_at === null);
+    const clamp = hasOpenRow ? await computeOutageClampTimestamp(knex) : 0;
+    return rows.map((row) => ({
+        startedAt: row.started_at,
+        endedAt: row.ended_at ?? Math.max(clamp, row.started_at),
+    }));
+}

package/lib/archive/db-ops/pages/read/build-page-query.js

@@ -36,5 +36,5 @@ export function buildPageQuery(knex) {
         .leftJoin('url_refs as og_image_ur', 'og_image_ur.id', 'pm.og_image_url_id')
         .leftJoin('url_refs as twitter_image_ur', 'twitter_image_ur.id', 'pm.twitter_image_url_id')
         .leftJoin('json_refs as extras_ref', 'extras_ref.id', 'pm.meta_extras_json_id')
-        .select('ci.id as id', 'ur.url as url', 'ci.redirect_dest_id as redirectDestId', 'ci.scraped as scraped', 'ci.is_target as isTarget', 'ci.is_external as isExternal', 'ci.status as status', 'ci.status_text as statusText', 'ctr.raw as contentType', 'ci.content_length as contentLength', 'ci.header_set_id as headerSetId', 'pm.lang as lang', 'pm.dir as dir', 'pm.charset as charset', 'pm.base_href as baseHref', 'pm.viewport_raw as viewport_raw', 'pm.theme_color as themeColor', 'pm.application_name as applicationName', 'pm.author as author', 'pm.generator as generator', 'pm.publisher as publisher', 'title_ref.text as title', 'description_ref.text as description', 'keywords_ref.text as keywords', 'robots_raw_ref.text as robots_raw', 'pm.robots_noindex as robots_noindex', 'pm.robots_nofollow as robots_nofollow', 'pm.robots_noarchive as robots_noarchive', 'pm.robots_noimageindex as robots_noimageindex', 'pm.googlebot as googlebot', 'canonical_ur.url as canonical', 'amphtml_ur.url as amphtml', 'manifest_ur.url as manifest', 'icon_ur.url as icon_href', 'apple_ur.url as appleTouchIcon_href', 'pm.og_type as og_type', 'og_title_ref.text as og_title', 'og_url_ur.url as og_url', 'pm.og_site_name as og_site_name', 'og_description_ref.text as og_description', 'og_image_ur.url as og_image', 'pm.og_image_alt as og_image_alt', 'pm.og_image_width as og_image_width', 'pm.og_image_height as og_image_height', 'pm.og_locale as og_locale', 'pm.og_article_published_time as og_article_published_time', 'pm.og_article_modified_time as og_article_modified_time', 'pm.twitter_card as twitter_card', 'pm.twitter_site as twitter_site', 'pm.twitter_creator as twitter_creator', 'twitter_title_ref.text as twitter_title', 'twitter_description_ref.text as twitter_description', 'twitter_image_ur.url as twitter_image', 'pm.fb_app_id as fb_app_id', 'pm.verification_google as verification_google', 'pm.format_detection_telephone as formatDetection_telephone', 'ci.first_crawled_at as firstCrawledAt', 'ci.last_crawled_at as lastCrawledAt', 'pm.tag_count as tag_count', 'pm.jsonld_count as jsonld_count', 'pm.tags_providers_csv as tags_providers_csv', 'extras_ref.json_text as extras_body', 'extras_ref.codec as extras_codec', 'ci.is_skipped as isSkipped', 'ci.skip_reason as skipReason', 'ci.crawl_order as order', 'ci.source as source');
+        .select('ci.id as id', 'ur.url as url', 'ci.redirect_dest_id as redirectDestId', 'ci.scraped as scraped', 'ci.is_target as isTarget', 'ci.is_external as isExternal', 'ci.status as status', 'ci.status_text as statusText', 'ctr.raw as contentType', 'ci.content_length as contentLength', 'ci.header_set_id as headerSetId', 'pm.lang as lang', 'pm.dir as dir', 'pm.charset as charset', 'pm.base_href as baseHref', 'pm.viewport_raw as viewport_raw', 'pm.theme_color as themeColor', 'pm.application_name as applicationName', 'pm.author as author', 'pm.generator as generator', 'pm.publisher as publisher', 'title_ref.text as title', 'description_ref.text as description', 'keywords_ref.text as keywords', 'robots_raw_ref.text as robots_raw', 'pm.robots_noindex as robots_noindex', 'pm.robots_nofollow as robots_nofollow', 'pm.robots_noarchive as robots_noarchive', 'pm.robots_noimageindex as robots_noimageindex', 'pm.googlebot as googlebot', 'canonical_ur.url as canonical', 'amphtml_ur.url as amphtml', 'manifest_ur.url as manifest', 'icon_ur.url as icon_href', 'apple_ur.url as appleTouchIcon_href', 'pm.og_type as og_type', 'og_title_ref.text as og_title', 'og_url_ur.url as og_url', 'pm.og_site_name as og_site_name', 'og_description_ref.text as og_description', 'og_image_ur.url as og_image', 'pm.og_image_alt as og_image_alt', 'pm.og_image_width as og_image_width', 'pm.og_image_height as og_image_height', 'pm.og_locale as og_locale', 'pm.og_article_published_time as og_article_published_time', 'pm.og_article_modified_time as og_article_modified_time', 'pm.twitter_card as twitter_card', 'pm.twitter_site as twitter_site', 'pm.twitter_creator as twitter_creator', 'twitter_title_ref.text as twitter_title', 'twitter_description_ref.text as twitter_description', 'twitter_image_ur.url as twitter_image', 'pm.fb_app_id as fb_app_id', 'pm.verification_google as verification_google', 'pm.format_detection_telephone as formatDetection_telephone', 'ci.first_crawled_at as firstCrawledAt', 'ci.last_crawled_at as lastCrawledAt', 'pm.tag_count as tag_count', 'pm.jsonld_count as jsonld_count', 'pm.tags_providers_csv as tags_providers_csv', 'pm.main_content_node_name as main_content_node_name', 'pm.main_content_id as main_content_id', 'pm.main_content_role as main_content_role', 'pm.main_content_selector as main_content_selector', 'pm.main_content_class_list as main_content_class_list', 'pm.main_content_word_count as main_content_word_count', 'pm.main_content_body_word_count as main_content_body_word_count', 'pm.main_content_heading_count as main_content_heading_count', 'pm.main_content_image_count as main_content_image_count', 'pm.main_content_table_count as main_content_table_count', 'pm.main_content_button_count as main_content_button_count', 'pm.main_content_iframe_count as main_content_iframe_count', 'pm.main_content_video_count as main_content_video_count', 'pm.main_content_audio_count as main_content_audio_count', 'pm.main_content_canvas_count as main_content_canvas_count', 'pm.scroll_height_desktop as scroll_height_desktop', 'pm.scroll_height_mobile as scroll_height_mobile', 'extras_ref.json_text as extras_body', 'extras_ref.codec as extras_codec', 'ci.is_skipped as isSkipped', 'ci.skip_reason as skipReason', 'ci.crawl_order as order', 'ci.source as source');
 }

package/lib/archive/db-ops/pages/reset/repromote-external-pages.js

@@ -65,19 +65,20 @@ export async function repromoteExternalPages(knex, scopes, options) {
             // demotion.
         });
         // Clear the prior crawl's data for the repromoted pages. `updatePage`
-        // also replaces anchor_edges/image_items/tags/jsonld when it
-        // re-scrapes them, but only when the new scrape is non-empty — so
-        // this pre-clear is still load-bearing for pages that get
+        // also replaces anchor_edges/image_items/tags/jsonld/page_main_content_*
+        // when it re-scrapes them, but only when the new scrape is non-empty —
+        // so this pre-clear is still load-bearing for pages that get
         // repromoted but then re-scrape to nothing (or are never reached
         // again), and it is the only place `resource_ref_edges` is cleared.
         // Deleting the `page_meta` row (rather than nulling every column)
-        // clears title / description / og:* / twitter:* / meta_extras in
-        // one statement; a re-scrape re-inserts it via
-        // `ON CONFLICT(page_id) DO UPDATE`. `page_tags` / `page_jsonld` are
-        // cleared explicitly even though both tables also carry ON DELETE
-        // CASCADE — we keep the existing pattern of explicit chunked
-        // DELETEs rather than relying on CASCADE indirectly (and would not
-        // cascade anyway: the parent `content_items` row is updated, not
+        // clears title / description / og:* / twitter:* / meta_extras /
+        // main_content_* in one statement; a re-scrape re-inserts it via
+        // `ON CONFLICT(page_id) DO UPDATE`. `page_tags` / `page_jsonld` /
+        // `page_main_content_*` are cleared explicitly even though all of
+        // them also carry ON DELETE CASCADE — we keep the existing pattern
+        // of explicit chunked DELETEs rather than relying on CASCADE
+        // indirectly (and would not cascade anyway: the parent
+        // `content_items` row is updated, not
         // deleted). Orphan blobs in `page_html_blobs` are left behind; #23
         // will add GC.
         await knex('page_meta').whereIn('page_id', chunk).delete();
@@ -87,6 +88,14 @@ export async function repromoteExternalPages(knex, scopes, options) {
         await knex('page_html_ref').whereIn('page_id', chunk).delete();
         await knex('page_tags').whereIn('pageId', chunk).delete();
         await knex('page_jsonld').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_headings').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_images').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_tables').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_buttons').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_iframes').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_videos').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_audios').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_canvases').whereIn('pageId', chunk).delete();
     }
     dbLog('Repromoted %d external pages back to pending', promotedUrls.length);
     return promotedUrls;

package/lib/archive/db-ops/pages/reset/reset-failed-pages.d.ts

@@ -29,15 +29,30 @@ import type { Knex } from 'knex';
  * target shrinking across `--retry-failed` passes by leaving deterministic
  * dead-ends alone.
  *
+ * **Outage override**: before applying the permanent-kind exclusion, the
+ * message's `createdAt` is checked against every recorded
+ * `network_outages` window (see `is-within-outage-window.ts`). A `dns` (or
+ * any other permanent-kind) failure whose timestamp falls inside a window
+ * is treated as retryable regardless — `dns` is only a permanent,
+ * site-specific verdict when nothing else explains it; inside a confirmed
+ * operator-network outage, the same `getaddrinfo ENOTFOUND` message is
+ * evidence about the CRAWLER's connectivity, not the target site, and
+ * excluding it from retry would strand a perfectly reachable host as a
+ * false permanent failure for the rest of the archive's life. An archive
+ * with no recorded outages (`listNetworkOutages` returns `[]`) behaves
+ * exactly as before this override existed.
+ *
  * Matching rows — internal and external alike — are demoted back to pending
  * (`scraped = 0`) and have their stale scrape metadata cleared (the
  * `page_meta` row is deleted outright rather than nulled column-by-column).
  * The page row itself is kept (id preserved) so existing
  * `anchor_edges.href_page_id` referrers stay valid, and `is_external` is
  * left untouched so the next pass re-classifies each page from the crawl
- * scope. Related `anchor_edges`, `image_items`, `resource_ref_edges`, and
- * `page_errors` rows are deleted so the re-scrape can re-insert fresh data
- * without duplicates.
+ * scope. Related `anchor_edges`, `image_items`, `resource_ref_edges`,
+ * `page_errors`, and the `page_main_content_*` child tables are deleted so
+ * the re-scrape can re-insert fresh data without duplicates — kept in sync
+ * with the `page_meta` row deletion above so a reset page's `main_content_*`
+ * counts and its child-table detail never disagree.
  *
  * SELECT and UPDATE/DELETE statements are chunked to stay below SQLite's
  * `SQLITE_LIMIT_VARIABLE_NUMBER`.

package/lib/archive/db-ops/pages/reset/reset-failed-pages.js

@@ -1,7 +1,9 @@
 import { classifyErrorKind } from '../../../../classify-error-kind.js';
+import { isWithinOutageWindow } from '../../../../is-within-outage-window.js';
 import { PERMANENT_ERROR_KINDS } from '../../../../permanent-error-kinds.js';
 import { dbLog } from '../../../debug.js';
 import { getFailedPageMessages } from '../../../get-failed-page-messages.js';
+import { listNetworkOutages } from '../../outages/list-network-outages.js';
 /**
  * Reset previously-attempted pages that ended in a recoverable failure so a
  * follow-up crawl can re-fetch them from scratch.
@@ -32,15 +34,30 @@ import { getFailedPageMessages } from '../../../get-failed-page-messages.js';
  * target shrinking across `--retry-failed` passes by leaving deterministic
  * dead-ends alone.
  *
+ * **Outage override**: before applying the permanent-kind exclusion, the
+ * message's `createdAt` is checked against every recorded
+ * `network_outages` window (see `is-within-outage-window.ts`). A `dns` (or
+ * any other permanent-kind) failure whose timestamp falls inside a window
+ * is treated as retryable regardless — `dns` is only a permanent,
+ * site-specific verdict when nothing else explains it; inside a confirmed
+ * operator-network outage, the same `getaddrinfo ENOTFOUND` message is
+ * evidence about the CRAWLER's connectivity, not the target site, and
+ * excluding it from retry would strand a perfectly reachable host as a
+ * false permanent failure for the rest of the archive's life. An archive
+ * with no recorded outages (`listNetworkOutages` returns `[]`) behaves
+ * exactly as before this override existed.
+ *
  * Matching rows — internal and external alike — are demoted back to pending
  * (`scraped = 0`) and have their stale scrape metadata cleared (the
  * `page_meta` row is deleted outright rather than nulled column-by-column).
  * The page row itself is kept (id preserved) so existing
  * `anchor_edges.href_page_id` referrers stay valid, and `is_external` is
  * left untouched so the next pass re-classifies each page from the crawl
- * scope. Related `anchor_edges`, `image_items`, `resource_ref_edges`, and
- * `page_errors` rows are deleted so the re-scrape can re-insert fresh data
- * without duplicates.
+ * scope. Related `anchor_edges`, `image_items`, `resource_ref_edges`,
+ * `page_errors`, and the `page_main_content_*` child tables are deleted so
+ * the re-scrape can re-insert fresh data without duplicates — kept in sync
+ * with the `page_meta` row deletion above so a reset page's `main_content_*`
+ * counts and its child-table detail never disagree.
  *
  * SELECT and UPDATE/DELETE statements are chunked to stay below SQLite's
  * `SQLITE_LIMIT_VARIABLE_NUMBER`.
@@ -67,16 +84,29 @@ export async function resetFailedPages(knex) {
     }
     const candidateIds = candidates.map((row) => row.id);
     const candidateUrls = candidates.map((row) => row.url);
-    const messages = await getFailedPageMessages(knex, candidateIds, candidateUrls);
-    // Drop candidates whose latest recorded message classifies as permanent.
-    // An empty/absent message stays in the retry pool — we keep retrying when
-    // we don't know it's permanent, erring on the side of investigation.
+    // Unrelated tables (page_errors/crawl_errors vs network_outages), no data
+    // dependency between them — run concurrently instead of paying two
+    // sequential round-trips on every `--retry-failed` pass.
+    const [messages, outageWindows] = await Promise.all([
+        getFailedPageMessages(knex, candidateIds, candidateUrls),
+        listNetworkOutages(knex),
+    ]);
+    // Drop candidates whose latest recorded message classifies as permanent —
+    // UNLESS that message's timestamp falls inside a recorded network outage,
+    // in which case the permanent-kind verdict is overridden (see the
+    // "Outage override" section of this function's docstring). An
+    // empty/absent message stays in the retry pool regardless — we keep
+    // retrying when we don't know it's permanent, erring on the side of
+    // investigation.
     const retryable = candidates.filter((row) => {
-        const message = messages.get(row.id) ?? '';
-        if (message === '') {
+        const resolved = messages.get(row.id);
+        if (resolved === undefined || resolved.message === '') {
+            return true;
+        }
+        if (!PERMANENT_ERROR_KINDS.has(classifyErrorKind(resolved.message))) {
             return true;
         }
-        return !PERMANENT_ERROR_KINDS.has(classifyErrorKind(message));
+        return isWithinOutageWindow(resolved.createdAt, outageWindows);
     });
     const excludedCount = candidates.length - retryable.length;
     if (excludedCount > 0) {
@@ -118,6 +148,14 @@ export async function resetFailedPages(knex) {
         await knex('page_html_ref').whereIn('page_id', chunk).delete();
         await knex('page_tags').whereIn('pageId', chunk).delete();
         await knex('page_jsonld').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_headings').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_images').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_tables').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_buttons').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_iframes').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_videos').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_audios').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_canvases').whereIn('pageId', chunk).delete();
     }
     dbLog('Reset %d failed pages back to pending', urls.length);
     return urls;

package/lib/archive/db-ops/pages/write/insert-audios.d.ts

@@ -0,0 +1,14 @@
+import type { MainContentsData } from '@d-zero/beholder';
+import type { Knex } from 'knex';
+/**
+ * Replaces the page's `page_main_content_audios` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export declare function insertAudios(pageId: number, mainContents: MainContentsData, trx: Knex.Transaction): Promise<void>;

package/lib/archive/db-ops/pages/write/insert-audios.js

@@ -0,0 +1,25 @@
+import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+/**
+ * Replaces the page's `page_main_content_audios` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export async function insertAudios(pageId, mainContents, trx) {
+    if (mainContents.audios.length === 0)
+        return;
+    const rows = mainContents.audios.map((audio, order) => ({
+        pageId,
+        order,
+        src: audio.src,
+    }));
+    await trx('page_main_content_audios').where('pageId', pageId).delete();
+    await eachSplitted(rows, 100, async (chunk) => {
+        await trx('page_main_content_audios').insert(chunk);
+    });
+}

package/lib/archive/db-ops/pages/write/insert-buttons.d.ts

@@ -0,0 +1,14 @@
+import type { MainContentsData } from '@d-zero/beholder';
+import type { Knex } from 'knex';
+/**
+ * Replaces the page's `page_main_content_buttons` rows with the freshly
+ * captured set. Called inside `updatePage`'s transaction, gated on
+ * `mainContents !== null`.
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does not
+ * wipe prior rows on a degraded re-scrape.
+ * @param pageId - The owning `content_items.id`.
+ * @param mainContents - Beholder's per-page main-content metrics.
+ * @param trx - The active transaction.
+ */
+export declare function insertButtons(pageId: number, mainContents: MainContentsData, trx: Knex.Transaction): Promise<void>;