@nitpicker/crawler 0.14.0 → 0.15.0

package/lib/archive/db-ops/console-logs/upsert-console-log-item.d.ts

@@ -0,0 +1,29 @@
+import type { ConsoleLogItemRow } from './types.js';
+import type { WriteRefCaches } from '../_shared/types.js';
+import type { Knex } from 'knex';
+/**
+ * Resolves the `console_log_items.id` for one console log entry's content,
+ * inserting the dictionary row when the hash is not yet known.
+ *
+ * Same no-op-update-for-`RETURNING` idiom as `upsertUrlRef` / `upsertJsonRef`:
+ * `INSERT ... ON CONFLICT(hash) DO UPDATE SET hash = hash RETURNING id`
+ * yields the existing row's id on a hash collision without a separate
+ * SELECT round trip, and (unlike `INSERT OR IGNORE ... RETURNING`) still
+ * returns a row when the insert itself is the no-op.
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ * @param caches - The connection's write-side id caches; mutated in place.
+ * @param row - The content hash plus every resolved ref id / scalar column.
+ * @returns The `console_log_items.id` of the existing or newly inserted row.
+ * @example
+ * const id = await upsertConsoleLogItem(trx, caches, {
+ *   hash: computeConsoleLogHash(entry),
+ *   type: entry.type,
+ *   textId,
+ *   argsJsonId: null,
+ *   locUrlId: null,
+ *   locLine: null,
+ *   locColumn: null,
+ *   stackTextId: null,
+ * });
+ */
+export declare function upsertConsoleLogItem(qb: Knex | Knex.Transaction, caches: WriteRefCaches, row: ConsoleLogItemRow): Promise<number>;

package/lib/archive/db-ops/console-logs/upsert-console-log-item.js

@@ -0,0 +1,52 @@
+/**
+ * Resolves the `console_log_items.id` for one console log entry's content,
+ * inserting the dictionary row when the hash is not yet known.
+ *
+ * Same no-op-update-for-`RETURNING` idiom as `upsertUrlRef` / `upsertJsonRef`:
+ * `INSERT ... ON CONFLICT(hash) DO UPDATE SET hash = hash RETURNING id`
+ * yields the existing row's id on a hash collision without a separate
+ * SELECT round trip, and (unlike `INSERT OR IGNORE ... RETURNING`) still
+ * returns a row when the insert itself is the no-op.
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ * @param caches - The connection's write-side id caches; mutated in place.
+ * @param row - The content hash plus every resolved ref id / scalar column.
+ * @returns The `console_log_items.id` of the existing or newly inserted row.
+ * @example
+ * const id = await upsertConsoleLogItem(trx, caches, {
+ *   hash: computeConsoleLogHash(entry),
+ *   type: entry.type,
+ *   textId,
+ *   argsJsonId: null,
+ *   locUrlId: null,
+ *   locLine: null,
+ *   locColumn: null,
+ *   stackTextId: null,
+ * });
+ */
+export async function upsertConsoleLogItem(qb, caches, row) {
+    const hex = row.hash.toString('hex');
+    const cached = caches.consoleLogIds.get(hex);
+    if (cached !== undefined) {
+        return cached;
+    }
+    const rows = await qb.raw(`INSERT INTO console_log_items
+			(hash, type, text_id, args_json_id, loc_url_id, loc_line, loc_column, stack_text_id)
+		 VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+		 ON CONFLICT(hash) DO UPDATE SET hash = hash
+		 RETURNING id`, [
+        row.hash,
+        row.type,
+        row.textId,
+        row.argsJsonId,
+        row.locUrlId,
+        row.locLine,
+        row.locColumn,
+        row.stackTextId,
+    ]);
+    const first = rows[0];
+    if (first === undefined) {
+        throw new Error('upsertConsoleLogItem: RETURNING yielded no row');
+    }
+    caches.consoleLogIds.set(hex, first.id);
+    return first.id;
+}

package/lib/archive/db-ops/errors/list-dns-burned-host-candidates.d.ts

@@ -16,6 +16,17 @@ import type { Knex } from 'knex';
  *
  * Returns `[]` on legacy archives that pre-date the `crawl_errors`
  * table — the `hasTable` guard keeps the call non-destructive.
+ *
+ * **Outage exclusion**: a host whose latest DNS error's `createdAt` falls
+ * inside a recorded `network_outages` window is dropped from the result
+ * regardless of the other checks. A host with no prior success this
+ * archive has ever seen (the case this whole exclusion-bag scheme cannot
+ * otherwise catch) would, without this check, get preload-seeded into
+ * `dnsBurnedHostCache` and short-circuit on EVERY subsequent session
+ * forever — the exact "damage 3" this function's caller
+ * (`#preloadDnsBurnedHostCache`) exists to prevent for hosts genuinely
+ * proven dead, not to inflict on hosts merely unlucky enough to be first
+ * contacted during an operator-side network blip.
  * @param knex - Knex query builder connected to the archive DB.
  * @returns Lower-cased hostnames safe to short-circuit.
  */

package/lib/archive/db-ops/errors/list-dns-burned-host-candidates.js

@@ -1,4 +1,6 @@
 import { classifyErrorKind } from '../../../classify-error-kind.js';
+import { isWithinOutageWindow } from '../../../is-within-outage-window.js';
+import { listNetworkOutages } from '../outages/list-network-outages.js';
 /**
  * Hostnames whose `crawl_errors` history is consistently DNS failures and
  * for which no recent 2xx-3xx page or resource is recorded — i.e. hosts
@@ -16,6 +18,17 @@ import { classifyErrorKind } from '../../../classify-error-kind.js';
  *
  * Returns `[]` on legacy archives that pre-date the `crawl_errors`
  * table — the `hasTable` guard keeps the call non-destructive.
+ *
+ * **Outage exclusion**: a host whose latest DNS error's `createdAt` falls
+ * inside a recorded `network_outages` window is dropped from the result
+ * regardless of the other checks. A host with no prior success this
+ * archive has ever seen (the case this whole exclusion-bag scheme cannot
+ * otherwise catch) would, without this check, get preload-seeded into
+ * `dnsBurnedHostCache` and short-circuit on EVERY subsequent session
+ * forever — the exact "damage 3" this function's caller
+ * (`#preloadDnsBurnedHostCache`) exists to prevent for hosts genuinely
+ * proven dead, not to inflict on hosts merely unlucky enough to be first
+ * contacted during an operator-side network blip.
  * @param knex - Knex query builder connected to the archive DB.
  * @returns Lower-cased hostnames safe to short-circuit.
  */
@@ -118,11 +131,12 @@ export async function listDnsBurnedHostCandidates(knex) {
         }
         resourceOkHosts.add(host);
     }
+    const outageWindows = await listNetworkOutages(knex);
     // A candidate host is burned only if neither pages nor resources hold a
-    // 2xx-3xx for it, AND its latest 2xx page (if any) is not newer than
-    // the latest DNS error. The third check guards against re-burning a
-    // host that recovered between the last DNS failure and the most recent
-    // crawl.
+    // 2xx-3xx for it, its latest 2xx page (if any) is not newer than the
+    // latest DNS error (guards against re-burning a host that recovered
+    // between the last DNS failure and the most recent crawl), AND that
+    // latest DNS error did not happen during a recorded network outage.
     const burned = [];
     for (const [host, latestErrorAt] of candidateLatestErrorAt) {
         if (pageOkHosts.has(host)) {
@@ -135,6 +149,9 @@ export async function listDnsBurnedHostCandidates(knex) {
         if (typeof latestOkAt === 'number' && latestOkAt > latestErrorAt) {
             continue;
         }
+        if (isWithinOutageWindow(latestErrorAt, outageWindows)) {
+            continue;
+        }
         burned.push(host);
     }
     return burned;

package/lib/archive/db-ops/inventory/record-inventory-run.js

@@ -27,6 +27,7 @@ export async function recordInventoryRun(knex, meta) {
         new_pages: meta.new_pages ?? null,
         new_resources: meta.new_resources ?? null,
         scope_skipped: meta.scope_skipped ?? null,
+        invalid_skipped: meta.invalid_skipped ?? null,
         notes: meta.notes ?? null,
     })
         .returning('id');

package/lib/archive/db-ops/lifecycle/init.d.ts

@@ -2,7 +2,7 @@ import type { Knex } from 'knex';
 /**
  * Initializes the database schema if tables do not exist, then runs the
  * remaining lightweight migrations (`info.roots`, `info.mainContentSelector`,
- * `page_meta.main_content_*`).
+ * `page_meta.main_content_*`, `inventory_runs.invalid_skipped`).
  *
  * There is deliberately no per-table *table-creation* migration chain here:
  * `assertCompatibleVersion` (called below, before any schema work) rejects
@@ -17,7 +17,16 @@ import type { Knex } from 'knex';
  * `CREATE TABLE IF NOT EXISTS` is a no-op once the table is present. Column
  * additions to an existing 0.13 table are therefore the one case that still
  * needs an explicit `hasColumn`-guarded `ALTER TABLE` here (`migrateInfoRoots`,
- * `migrateMainContentsColumns`) rather than a DDL-string change alone.
+ * `migrateMainContentsColumns`, `migratePageMetaBodyHash`,
+ * `migratePageMetaConsoleErrorCount`, `migrateContentItemsAliasOfId`,
+ * `migrateInventoryRunsInvalidSkipped`) rather than a DDL-string change alone.
+ *
+ * `closeStaleOpenNetworkOutages` is not a schema migration (no columns
+ * change) but belongs at this same boot phase for the same reason the
+ * others do: it must run before ANY reader (`resetFailedPages`,
+ * `listDnsBurnedHostCandidates`, …) can observe a `network_outages` row
+ * left `ended_at = NULL` by a crawl process that was killed mid-outage in a
+ * prior session.
  *
  * In read-only mode schema init + migration are SKIPPED so the same DB
  * can be opened safely by a viewer attached to a live (or interrupted)

package/lib/archive/db-ops/lifecycle/init.js

@@ -1,12 +1,17 @@
 import { applyConnectionPragmas, initSchema } from '../../init-schema.js';
 import { assertCompatibleVersion } from '../../meta/assert-compatible-version.js';
+import { migrateContentItemsAliasOfId } from '../../migrate-content-items-alias-of-id.js';
 import { migrateInfoMainContentSelector } from '../../migrate-info-main-content-selector.js';
 import { migrateInfoRoots } from '../../migrate-info-roots.js';
+import { migrateInventoryRunsInvalidSkipped } from '../../migrate-inventory-runs-invalid-skipped.js';
 import { migrateMainContentsColumns } from '../../migrate-main-contents-columns.js';
+import { migratePageMetaBodyHash } from '../../migrate-page-meta-body-hash.js';
+import { migratePageMetaConsoleErrorCount } from '../../migrate-page-meta-console-error-count.js';
+import { closeStaleOpenNetworkOutages } from '../outages/close-stale-open-network-outages.js';
 /**
  * Initializes the database schema if tables do not exist, then runs the
  * remaining lightweight migrations (`info.roots`, `info.mainContentSelector`,
- * `page_meta.main_content_*`).
+ * `page_meta.main_content_*`, `inventory_runs.invalid_skipped`).
  *
  * There is deliberately no per-table *table-creation* migration chain here:
  * `assertCompatibleVersion` (called below, before any schema work) rejects
@@ -21,7 +26,16 @@ import { migrateMainContentsColumns } from '../../migrate-main-contents-columns.
  * `CREATE TABLE IF NOT EXISTS` is a no-op once the table is present. Column
  * additions to an existing 0.13 table are therefore the one case that still
  * needs an explicit `hasColumn`-guarded `ALTER TABLE` here (`migrateInfoRoots`,
- * `migrateMainContentsColumns`) rather than a DDL-string change alone.
+ * `migrateMainContentsColumns`, `migratePageMetaBodyHash`,
+ * `migratePageMetaConsoleErrorCount`, `migrateContentItemsAliasOfId`,
+ * `migrateInventoryRunsInvalidSkipped`) rather than a DDL-string change alone.
+ *
+ * `closeStaleOpenNetworkOutages` is not a schema migration (no columns
+ * change) but belongs at this same boot phase for the same reason the
+ * others do: it must run before ANY reader (`resetFailedPages`,
+ * `listDnsBurnedHostCandidates`, …) can observe a `network_outages` row
+ * left `ended_at = NULL` by a crawl process that was killed mid-outage in a
+ * prior session.
  *
  * In read-only mode schema init + migration are SKIPPED so the same DB
  * can be opened safely by a viewer attached to a live (or interrupted)
@@ -49,4 +63,9 @@ export async function init(knex, readOnly) {
     await migrateInfoRoots(knex);
     await migrateInfoMainContentSelector(knex);
     await migrateMainContentsColumns(knex);
+    await migratePageMetaBodyHash(knex);
+    await migratePageMetaConsoleErrorCount(knex);
+    await migrateContentItemsAliasOfId(knex);
+    await migrateInventoryRunsInvalidSkipped(knex);
+    await closeStaleOpenNetworkOutages(knex);
 }

package/lib/archive/db-ops/outages/close-network-outage.d.ts

@@ -0,0 +1,12 @@
+import type { Knex } from 'knex';
+/**
+ * Close an outage row by stamping `ended_at` — but ONLY if it is still
+ * open. The `whereNull('ended_at')` guard is what makes this idempotent: a
+ * second call (e.g. a duplicate recovery-probe success racing the first)
+ * matches zero rows and is a silent no-op, rather than overwriting an
+ * already-recorded `ended_at` with a later timestamp.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param id - The `network_outages.id` to close.
+ * @param endedAt - Epoch ms the outage is considered to have ended.
+ */
+export declare function closeNetworkOutage(knex: Knex, id: number, endedAt: number): Promise<void>;

package/lib/archive/db-ops/outages/close-network-outage.js

@@ -0,0 +1,15 @@
+/**
+ * Close an outage row by stamping `ended_at` — but ONLY if it is still
+ * open. The `whereNull('ended_at')` guard is what makes this idempotent: a
+ * second call (e.g. a duplicate recovery-probe success racing the first)
+ * matches zero rows and is a silent no-op, rather than overwriting an
+ * already-recorded `ended_at` with a later timestamp.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param id - The `network_outages.id` to close.
+ * @param endedAt - Epoch ms the outage is considered to have ended.
+ */
+export async function closeNetworkOutage(knex, id, endedAt) {
+    await knex('network_outages').where({ id }).whereNull('ended_at').update({
+        ended_at: endedAt,
+    });
+}

package/lib/archive/db-ops/outages/close-stale-open-network-outages.d.ts

@@ -0,0 +1,26 @@
+import type { Knex } from 'knex';
+/**
+ * Durably close every `network_outages` row still `ended_at = NULL` at the
+ * start of a writer session.
+ *
+ * A row is left open only when the crawl process was killed (Ctrl-C / OOM
+ * / SIGKILL) mid-outage, before a recovery probe could close it. Called
+ * once from `db-ops/lifecycle/init.ts` — right after `initSchema` (which
+ * guarantees the table exists) and before anything else touches
+ * `network_outages` — so that by the time `resetFailedPages` /
+ * `listDnsBurnedHostCandidates` / any other reader runs, no row can still
+ * be open from a PRIOR session. (A row opened by THIS session cannot exist
+ * yet at this point in the boot sequence — the sliding-window detector
+ * only starts once the crawl loop begins.)
+ *
+ * This complements, but does not replace, `list-network-outages.ts`'s
+ * on-the-fly clamp: that read-side resolution stays in place as a
+ * defensive fallback, but after this runs there should be nothing left for
+ * it to resolve.
+ *
+ * Idempotent: a row closed by a previous call (or by a normal
+ * recovery-probe success) is simply absent from the `whereNull('ended_at')`
+ * scan, so re-running this on every writer open is always safe.
+ * @param knex - Knex query builder connected to the archive DB.
+ */
+export declare function closeStaleOpenNetworkOutages(knex: Knex): Promise<void>;

package/lib/archive/db-ops/outages/close-stale-open-network-outages.js

@@ -0,0 +1,42 @@
+import { closeNetworkOutage } from './close-network-outage.js';
+import { computeOutageClampTimestamp } from './compute-outage-clamp-timestamp.js';
+/**
+ * Durably close every `network_outages` row still `ended_at = NULL` at the
+ * start of a writer session.
+ *
+ * A row is left open only when the crawl process was killed (Ctrl-C / OOM
+ * / SIGKILL) mid-outage, before a recovery probe could close it. Called
+ * once from `db-ops/lifecycle/init.ts` — right after `initSchema` (which
+ * guarantees the table exists) and before anything else touches
+ * `network_outages` — so that by the time `resetFailedPages` /
+ * `listDnsBurnedHostCandidates` / any other reader runs, no row can still
+ * be open from a PRIOR session. (A row opened by THIS session cannot exist
+ * yet at this point in the boot sequence — the sliding-window detector
+ * only starts once the crawl loop begins.)
+ *
+ * This complements, but does not replace, `list-network-outages.ts`'s
+ * on-the-fly clamp: that read-side resolution stays in place as a
+ * defensive fallback, but after this runs there should be nothing left for
+ * it to resolve.
+ *
+ * Idempotent: a row closed by a previous call (or by a normal
+ * recovery-probe success) is simply absent from the `whereNull('ended_at')`
+ * scan, so re-running this on every writer open is always safe.
+ * @param knex - Knex query builder connected to the archive DB.
+ */
+export async function closeStaleOpenNetworkOutages(knex) {
+    const hasTable = await knex.schema.hasTable('network_outages');
+    if (!hasTable) {
+        return;
+    }
+    const openRows = (await knex('network_outages')
+        .whereNull('ended_at')
+        .select('id', 'started_at'));
+    if (openRows.length === 0) {
+        return;
+    }
+    const clamp = await computeOutageClampTimestamp(knex);
+    for (const row of openRows) {
+        await closeNetworkOutage(knex, row.id, Math.max(clamp, row.started_at));
+    }
+}

package/lib/archive/db-ops/outages/compute-outage-clamp-timestamp.d.ts

@@ -0,0 +1,26 @@
+import type { Knex } from 'knex';
+/**
+ * Compute the timestamp used to resolve an outage row whose `ended_at` is
+ * still `NULL` (the crawl process was killed before a recovery probe could
+ * close it) into a concrete, bounded window.
+ *
+ * The archive's own most-recent observations — the latest `crawl_errors`
+ * timestamp and the latest `content_items.last_crawled_at` — are the only
+ * evidence available for "when did activity in this archive last happen",
+ * so the later of the two stands in for "the outage cannot have lasted
+ * past this point, because the archive shows activity here". Using the
+ * LARGER of the two (not just `crawl_errors`) matters because a session
+ * that crashed mid-outage may have its last successful page write be more
+ * recent than its last recorded error, or vice versa.
+ *
+ * Both source tables are guaranteed to exist by the time this runs — this
+ * helper is only ever called from writer-context code (see
+ * `list-network-outages.ts` / the boot-time stale-open finalizer), and
+ * `initSchema` creates both `crawl_errors` and `content_items` before any
+ * writer session's first query.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The larger of `MAX(crawl_errors.createdAt)` and
+ *   `MAX(content_items.last_crawled_at)`, or `0` if the archive has neither
+ *   (a brand-new archive with no activity yet).
+ */
+export declare function computeOutageClampTimestamp(knex: Knex): Promise<number>;

package/lib/archive/db-ops/outages/compute-outage-clamp-timestamp.js

@@ -0,0 +1,31 @@
+/**
+ * Compute the timestamp used to resolve an outage row whose `ended_at` is
+ * still `NULL` (the crawl process was killed before a recovery probe could
+ * close it) into a concrete, bounded window.
+ *
+ * The archive's own most-recent observations — the latest `crawl_errors`
+ * timestamp and the latest `content_items.last_crawled_at` — are the only
+ * evidence available for "when did activity in this archive last happen",
+ * so the later of the two stands in for "the outage cannot have lasted
+ * past this point, because the archive shows activity here". Using the
+ * LARGER of the two (not just `crawl_errors`) matters because a session
+ * that crashed mid-outage may have its last successful page write be more
+ * recent than its last recorded error, or vice versa.
+ *
+ * Both source tables are guaranteed to exist by the time this runs — this
+ * helper is only ever called from writer-context code (see
+ * `list-network-outages.ts` / the boot-time stale-open finalizer), and
+ * `initSchema` creates both `crawl_errors` and `content_items` before any
+ * writer session's first query.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The larger of `MAX(crawl_errors.createdAt)` and
+ *   `MAX(content_items.last_crawled_at)`, or `0` if the archive has neither
+ *   (a brand-new archive with no activity yet).
+ */
+export async function computeOutageClampTimestamp(knex) {
+    const [crawlErrorsRow] = (await knex('crawl_errors').max('createdAt as maxCreatedAt'));
+    const [contentItemsRow] = (await knex('content_items').max('last_crawled_at as maxLastCrawledAt'));
+    const latestError = crawlErrorsRow?.maxCreatedAt ?? 0;
+    const latestCrawl = contentItemsRow?.maxLastCrawledAt ?? 0;
+    return Math.max(latestError, latestCrawl);
+}

package/lib/archive/db-ops/outages/insert-network-outage.d.ts

@@ -0,0 +1,14 @@
+import type { InsertNetworkOutageParams } from '../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Appends one row to the `network_outages` journal, with `ended_at` left
+ * `NULL` — the row starts life as an open outage.
+ *
+ * Called the moment a recovery probe CONFIRMS a suspect outage (i.e. the
+ * probe itself failed, not merely the sliding-window threshold trip). See
+ * `close-network-outage.ts` for how the row is later closed.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param params - The confirmed-outage fields to record.
+ * @returns The autoincremented `id` of the newly-inserted row.
+ */
+export declare function insertNetworkOutage(knex: Knex, params: InsertNetworkOutageParams): Promise<number>;

package/lib/archive/db-ops/outages/insert-network-outage.js

@@ -0,0 +1,29 @@
+/**
+ * Appends one row to the `network_outages` journal, with `ended_at` left
+ * `NULL` — the row starts life as an open outage.
+ *
+ * Called the moment a recovery probe CONFIRMS a suspect outage (i.e. the
+ * probe itself failed, not merely the sliding-window threshold trip). See
+ * `close-network-outage.ts` for how the row is later closed.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param params - The confirmed-outage fields to record.
+ * @returns The autoincremented `id` of the newly-inserted row.
+ */
+export async function insertNetworkOutage(knex, params) {
+    const inserted = await knex
+        .from('network_outages')
+        .insert({
+        started_at: params.startedAt,
+        detected_at: params.detectedAt,
+        ended_at: null,
+        probe_host: params.probeHost,
+        trigger_error_count: params.triggerErrorCount,
+        trigger_host_count: params.triggerHostCount,
+    })
+        .returning('id');
+    const id = inserted[0]?.id;
+    if (typeof id !== 'number') {
+        throw new TypeError('insertNetworkOutage: INSERT returned no row id');
+    }
+    return id;
+}

package/lib/archive/db-ops/outages/list-network-outages.d.ts

@@ -0,0 +1,28 @@
+import type { OutageWindow } from '../../../is-within-outage-window.js';
+import type { Knex } from 'knex';
+/**
+ * List every recorded outage as a resolved (closed) {@link OutageWindow},
+ * suitable for `isWithinOutageWindow`.
+ *
+ * This is the crawler-internal counterpart to `@nitpicker/query`'s richer
+ * `listNetworkOutages` (full row shape, for CLI/MCP display) — this one
+ * exists purely to feed the three write-path consumers
+ * (`resetFailedPages`, `listDnsBurnedHostCandidates`, the gate itself) that
+ * only need "was this timestamp inside an outage", never the row's other
+ * columns.
+ *
+ * Any row whose `ended_at` is still `NULL` (the crawl process was killed
+ * before recovery) is resolved on the fly via
+ * {@link computeOutageClampTimestamp} rather than ever being returned as an
+ * unbounded window — see `is-within-outage-window.ts`'s `OutageWindow`
+ * docstring for why an open-ended window would be a correctness bug (every
+ * later error would retroactively read as network-caused, forever). This
+ * on-the-fly resolution does NOT persist to the row — it is a defensive
+ * fallback independent of whichever boot-time finalizer durably closes
+ * stale-open rows.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns Resolved outage windows. Empty when the archive predates the
+ *   `network_outages` table (self-healed on next writer open, so this is
+ *   never a permanent state) or has recorded no outages.
+ */
+export declare function listNetworkOutages(knex: Knex): Promise<OutageWindow[]>;

package/lib/archive/db-ops/outages/list-network-outages.js

@@ -0,0 +1,42 @@
+import { computeOutageClampTimestamp } from './compute-outage-clamp-timestamp.js';
+/**
+ * List every recorded outage as a resolved (closed) {@link OutageWindow},
+ * suitable for `isWithinOutageWindow`.
+ *
+ * This is the crawler-internal counterpart to `@nitpicker/query`'s richer
+ * `listNetworkOutages` (full row shape, for CLI/MCP display) — this one
+ * exists purely to feed the three write-path consumers
+ * (`resetFailedPages`, `listDnsBurnedHostCandidates`, the gate itself) that
+ * only need "was this timestamp inside an outage", never the row's other
+ * columns.
+ *
+ * Any row whose `ended_at` is still `NULL` (the crawl process was killed
+ * before recovery) is resolved on the fly via
+ * {@link computeOutageClampTimestamp} rather than ever being returned as an
+ * unbounded window — see `is-within-outage-window.ts`'s `OutageWindow`
+ * docstring for why an open-ended window would be a correctness bug (every
+ * later error would retroactively read as network-caused, forever). This
+ * on-the-fly resolution does NOT persist to the row — it is a defensive
+ * fallback independent of whichever boot-time finalizer durably closes
+ * stale-open rows.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns Resolved outage windows. Empty when the archive predates the
+ *   `network_outages` table (self-healed on next writer open, so this is
+ *   never a permanent state) or has recorded no outages.
+ */
+export async function listNetworkOutages(knex) {
+    const hasTable = await knex.schema.hasTable('network_outages');
+    if (!hasTable) {
+        return [];
+    }
+    const rows = (await knex('network_outages').select('started_at', 'ended_at'));
+    if (rows.length === 0) {
+        return [];
+    }
+    const hasOpenRow = rows.some((row) => row.ended_at === null);
+    const clamp = hasOpenRow ? await computeOutageClampTimestamp(knex) : 0;
+    return rows.map((row) => ({
+        startedAt: row.started_at,
+        endedAt: row.ended_at ?? Math.max(clamp, row.started_at),
+    }));
+}

package/lib/archive/db-ops/pages/reset/reset-failed-pages.d.ts

@@ -29,6 +29,19 @@ import type { Knex } from 'knex';
  * target shrinking across `--retry-failed` passes by leaving deterministic
  * dead-ends alone.
  *
+ * **Outage override**: before applying the permanent-kind exclusion, the
+ * message's `createdAt` is checked against every recorded
+ * `network_outages` window (see `is-within-outage-window.ts`). A `dns` (or
+ * any other permanent-kind) failure whose timestamp falls inside a window
+ * is treated as retryable regardless — `dns` is only a permanent,
+ * site-specific verdict when nothing else explains it; inside a confirmed
+ * operator-network outage, the same `getaddrinfo ENOTFOUND` message is
+ * evidence about the CRAWLER's connectivity, not the target site, and
+ * excluding it from retry would strand a perfectly reachable host as a
+ * false permanent failure for the rest of the archive's life. An archive
+ * with no recorded outages (`listNetworkOutages` returns `[]`) behaves
+ * exactly as before this override existed.
+ *
  * Matching rows — internal and external alike — are demoted back to pending
  * (`scraped = 0`) and have their stale scrape metadata cleared (the
  * `page_meta` row is deleted outright rather than nulled column-by-column).

package/lib/archive/db-ops/pages/reset/reset-failed-pages.js

@@ -1,7 +1,9 @@
 import { classifyErrorKind } from '../../../../classify-error-kind.js';
+import { isWithinOutageWindow } from '../../../../is-within-outage-window.js';
 import { PERMANENT_ERROR_KINDS } from '../../../../permanent-error-kinds.js';
 import { dbLog } from '../../../debug.js';
 import { getFailedPageMessages } from '../../../get-failed-page-messages.js';
+import { listNetworkOutages } from '../../outages/list-network-outages.js';
 /**
  * Reset previously-attempted pages that ended in a recoverable failure so a
  * follow-up crawl can re-fetch them from scratch.
@@ -32,6 +34,19 @@ import { getFailedPageMessages } from '../../../get-failed-page-messages.js';
  * target shrinking across `--retry-failed` passes by leaving deterministic
  * dead-ends alone.
  *
+ * **Outage override**: before applying the permanent-kind exclusion, the
+ * message's `createdAt` is checked against every recorded
+ * `network_outages` window (see `is-within-outage-window.ts`). A `dns` (or
+ * any other permanent-kind) failure whose timestamp falls inside a window
+ * is treated as retryable regardless — `dns` is only a permanent,
+ * site-specific verdict when nothing else explains it; inside a confirmed
+ * operator-network outage, the same `getaddrinfo ENOTFOUND` message is
+ * evidence about the CRAWLER's connectivity, not the target site, and
+ * excluding it from retry would strand a perfectly reachable host as a
+ * false permanent failure for the rest of the archive's life. An archive
+ * with no recorded outages (`listNetworkOutages` returns `[]`) behaves
+ * exactly as before this override existed.
+ *
  * Matching rows — internal and external alike — are demoted back to pending
  * (`scraped = 0`) and have their stale scrape metadata cleared (the
  * `page_meta` row is deleted outright rather than nulled column-by-column).
@@ -69,16 +84,29 @@ export async function resetFailedPages(knex) {
     }
     const candidateIds = candidates.map((row) => row.id);
     const candidateUrls = candidates.map((row) => row.url);
-    const messages = await getFailedPageMessages(knex, candidateIds, candidateUrls);
-    // Drop candidates whose latest recorded message classifies as permanent.
-    // An empty/absent message stays in the retry pool — we keep retrying when
-    // we don't know it's permanent, erring on the side of investigation.
+    // Unrelated tables (page_errors/crawl_errors vs network_outages), no data
+    // dependency between them — run concurrently instead of paying two
+    // sequential round-trips on every `--retry-failed` pass.
+    const [messages, outageWindows] = await Promise.all([
+        getFailedPageMessages(knex, candidateIds, candidateUrls),
+        listNetworkOutages(knex),
+    ]);
+    // Drop candidates whose latest recorded message classifies as permanent —
+    // UNLESS that message's timestamp falls inside a recorded network outage,
+    // in which case the permanent-kind verdict is overridden (see the
+    // "Outage override" section of this function's docstring). An
+    // empty/absent message stays in the retry pool regardless — we keep
+    // retrying when we don't know it's permanent, erring on the side of
+    // investigation.
     const retryable = candidates.filter((row) => {
-        const message = messages.get(row.id) ?? '';
-        if (message === '') {
+        const resolved = messages.get(row.id);
+        if (resolved === undefined || resolved.message === '') {
+            return true;
+        }
+        if (!PERMANENT_ERROR_KINDS.has(classifyErrorKind(resolved.message))) {
             return true;
         }
-        return !PERMANENT_ERROR_KINDS.has(classifyErrorKind(message));
+        return isWithinOutageWindow(resolved.createdAt, outageWindows);
     });
     const excludedCount = candidates.length - retryable.length;
     if (excludedCount > 0) {

package/lib/archive/db-ops/pages/write/update-page.js

@@ -1,6 +1,7 @@
 import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
 import { isHtmlContentType } from '../../../../crawler/is-html-content-type.js';
 import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+import { computeBodyHash } from '../../../body-hash/compute-body-hash.js';
 import { dbLog } from '../../../debug.js';
 import { deriveLineageFromParent } from '../../../derive-lineage-from-parent.js';
 import { matchImagesToDomPaths } from '../../../populate-entity-tables/match-images-to-dom-paths.js';
@@ -141,6 +142,12 @@ async function updatePageInTransaction(trx, knex, caches, page, destUrlObject, s
     // content check alone expresses the intent without a redundant term.
     if (writeHtml && page.html.length > 0) {
         await writePageHtmlBlob(pageId, page.html, trx);
+        // Computed from the same `page.html` written above, in the same
+        // transaction, so `page_meta.body_hash` never observably lags the
+        // snapshot it was derived from.
+        await trx('page_meta')
+            .where('page_id', pageId)
+            .update({ body_hash: computeBodyHash(page.html) });
     }
     else if (writeHtml &&
         page.contentType !== null &&
@@ -155,7 +162,12 @@ async function updatePageInTransaction(trx, knex, caches, page, destUrlObject, s
         // stale ref can only have been written by a snapshot-capable call
         // (`setPage`); `setExternalPage` passes `writeHtml = false` and never
         // sets `html`, so it has nothing to clear.
+        //
+        // `page_meta.body_hash` is cleared alongside it for the same reason —
+        // it is derived from the same HTML `page_html_ref` points at, so it
+        // must not outlive the ref it was computed from.
         await trx('page_html_ref').where('page_id', pageId).delete();
+        await trx('page_meta').where('page_id', pageId).update({ body_hash: null });
     }
     // Re-scrape semantics: the same URL can be scraped more than once
     // (e.g. `crawl --resume`, re-visits, `--append` re-promotion). Edges