@nitpicker/crawler 0.14.0 → 0.16.0

package/lib/archive/db-ops/console-logs/replace-console-logs.js

@@ -0,0 +1,123 @@
+import { upsertTextRefs } from '../../populate-entity-tables/upsert-text-refs.js';
+import { resolveRedirectChain } from '../../resolve-redirect-chain.js';
+import { clearWriteRefCaches } from '../_shared/clear-write-ref-caches.js';
+import { resolveContentItemId } from '../_shared/resolve-content-item-id.js';
+import { upsertJsonRef } from '../_shared/upsert-json-ref.js';
+import { upsertUrlRef } from '../_shared/upsert-url-ref.js';
+import { computeConsoleLogHash } from './compute-console-log-hash.js';
+import { stringifyConsoleLogArgs } from './stringify-console-log-args.js';
+import { upsertConsoleLogItem } from './upsert-console-log-item.js';
+/** Rows sent per `INSERT INTO page_console_logs ... VALUES (...)` statement. */
+const INSERT_CHUNK_SIZE = 500;
+/**
+ * Replaces one page's `page_console_logs` rows wholesale with a freshly
+ * captured set of console messages / page errors (issue #228).
+ *
+ * Scoped-Replace, the same pattern as `replaceAnchorEdges` /
+ * `replaceImageItems`: the page's existing rows are deleted and the new
+ * set is inserted in the same transaction, so a re-scrape (`--append` /
+ * `--retry-failed` / re-render) never accumulates duplicate occurrences.
+ * Unlike those two, there is no non-empty guard on the DELETE here — the
+ * caller (`Crawler#handleConsoleLogs`) already skips emitting the event
+ * entirely when `entries` is empty, which is what keeps a degraded
+ * re-scrape from wiping out a prior good capture.
+ *
+ * `pageUrl` is the originally-requested URL (matching `updatePage`'s
+ * `page.url.withoutHashAndAuth` contract), not necessarily the page that
+ * ends up holding the content: `resolveRedirectChain` derives the same
+ * redirect destination `updatePage` writes content under, so console logs
+ * attach to the row that actually carries the page's `page_meta`, not to
+ * an empty placeholder for the pre-redirect URL.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param pageUrl - The originally-requested URL, normalised
+ *   (`withoutHashAndAuth` form).
+ * @param redirectPaths - The redirect chain hops captured during fetch, in
+ *   order (empty when the page was not redirected).
+ * @param entries - The console log entries to persist. Must be non-empty —
+ *   callers should skip calling this function entirely for an empty list.
+ * @example
+ * await replaceConsoleLogs(knex, caches, page.url.withoutHashAndAuth, page.redirectPaths, entries);
+ */
+export async function replaceConsoleLogs(knex, caches, pageUrl, redirectPaths, entries) {
+    const { destUrl } = resolveRedirectChain(pageUrl, redirectPaths);
+    try {
+        await knex.transaction(async (trx) => {
+            const pageId = await resolveContentItemId(trx, caches, destUrl);
+            await trx('page_console_logs').where('pageId', pageId).delete();
+            const texts = new Set();
+            for (const entry of entries) {
+                // `text_refs` never stores the empty string (its upsert
+                // treats `''` as "nothing to dedupe" and skips it) — a
+                // `console.log()` call with zero arguments reports `text:
+                // ''`, so that entry resolves to `textId: null` below
+                // instead of looking it up.
+                if (entry.text !== '') {
+                    texts.add(entry.text);
+                }
+                if (entry.stack) {
+                    texts.add(entry.stack);
+                }
+            }
+            const textIds = await upsertTextRefs(trx, texts);
+            const rows = [];
+            for (const entry of entries) {
+                let textId = null;
+                if (entry.text !== '') {
+                    const resolved = textIds.get(entry.text);
+                    if (resolved === undefined) {
+                        throw new Error(`replaceConsoleLogs: text_refs id not resolved for "${entry.text}"`);
+                    }
+                    textId = resolved;
+                }
+                const stackTextId = entry.stack ? (textIds.get(entry.stack) ?? null) : null;
+                const argsJson = stringifyConsoleLogArgs(entry.args);
+                const argsJsonId = argsJson === null ? null : await upsertJsonRef(trx, caches, argsJson);
+                const locUrlId = entry.location?.url === undefined
+                    ? null
+                    : await upsertUrlRef(trx, caches, entry.location.url);
+                const consoleLogId = await upsertConsoleLogItem(trx, caches, {
+                    hash: computeConsoleLogHash({
+                        type: entry.type,
+                        text: entry.text,
+                        argsJson,
+                        location: entry.location,
+                        stack: entry.stack,
+                    }),
+                    type: entry.type,
+                    textId,
+                    argsJsonId,
+                    locUrlId,
+                    locLine: entry.location?.lineNumber ?? null,
+                    locColumn: entry.location?.columnNumber ?? null,
+                    stackTextId,
+                });
+                rows.push({ pageId, consoleLogId, ts: entry.ts });
+            }
+            for (let index = 0; index < rows.length; index += INSERT_CHUNK_SIZE) {
+                await trx('page_console_logs').insert(rows.slice(index, index + INSERT_CHUNK_SIZE));
+            }
+            // Denormalised onto `page_meta` (the same write-once-at-scrape-time
+            // pattern as `tag_count` / `jsonld_count`) so the Pages list can
+            // display and sort by it without a live JOIN + COUNT per row.
+            // Computed from `entries` directly rather than re-querying the rows
+            // just inserted above — the data is already in hand. A `page_meta`
+            // row may not exist yet for a `'skipped'` / `'error'` scrape; the
+            // UPDATE then simply affects zero rows rather than throwing.
+            const errorCount = entries.filter((entry) => entry.type === 'error' || entry.type === 'pageerror').length;
+            await trx('page_meta').where('page_id', pageId).update({
+                console_error_count: errorCount,
+            });
+        });
+    }
+    catch (error) {
+        // A rolled-back transaction can leave ids cached that no longer
+        // correspond to any row (AUTOINCREMENT never rewinds) — same
+        // cache-poisoning hazard `updatePage` / `recordRedirect` guard
+        // against. A full clear (not a partial one) is required because
+        // this function shares `caches` with every other write path on
+        // the same connection.
+        clearWriteRefCaches(caches);
+        throw error;
+    }
+}

package/lib/archive/db-ops/console-logs/stringify-console-log-args.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Serializes a console call's argument array to a JSON string, or `null`
+ * when there is nothing worth storing.
+ *
+ * Two cases collapse to `null` rather than an empty/degenerate string:
+ * an empty `args` array (most commonly a `"pageerror"` entry, which
+ * beholder always reports with `args: []`) and a `JSON.stringify` failure
+ * (e.g. a circular reference in a logged object) — both mean "no
+ * additional structured payload beyond `text`", not "a malformed one".
+ * @param args - The `ConsoleLogEntry.args` array to serialize.
+ * @returns The JSON string, or `null` when empty or unserializable.
+ * @example
+ * stringifyConsoleLogArgs(['a', 1]); // '["a",1]'
+ * stringifyConsoleLogArgs([]); // null
+ */
+export declare function stringifyConsoleLogArgs(args: readonly unknown[]): string | null;

package/lib/archive/db-ops/console-logs/stringify-console-log-args.js

@@ -0,0 +1,26 @@
+/**
+ * Serializes a console call's argument array to a JSON string, or `null`
+ * when there is nothing worth storing.
+ *
+ * Two cases collapse to `null` rather than an empty/degenerate string:
+ * an empty `args` array (most commonly a `"pageerror"` entry, which
+ * beholder always reports with `args: []`) and a `JSON.stringify` failure
+ * (e.g. a circular reference in a logged object) — both mean "no
+ * additional structured payload beyond `text`", not "a malformed one".
+ * @param args - The `ConsoleLogEntry.args` array to serialize.
+ * @returns The JSON string, or `null` when empty or unserializable.
+ * @example
+ * stringifyConsoleLogArgs(['a', 1]); // '["a",1]'
+ * stringifyConsoleLogArgs([]); // null
+ */
+export function stringifyConsoleLogArgs(args) {
+    if (args.length === 0) {
+        return null;
+    }
+    try {
+        return JSON.stringify(args);
+    }
+    catch {
+        return null;
+    }
+}

package/lib/archive/db-ops/console-logs/types.d.ts

@@ -0,0 +1,38 @@
+/**
+ * The content fields of a `ConsoleLogEntry` that determine its identity in
+ * `console_log_items` — everything except `pageUrl` / `ts`, which are
+ * per-occurrence rather than per-content (see `page_console_logs`).
+ */
+export interface ConsoleLogContent {
+    type: string;
+    text: string;
+    /**
+     * Pre-stringified `args` (via `stringifyConsoleLogArgs`), or `null` for
+     * an empty/unserializable args array. Callers compute this once and
+     * reuse it both for hashing here and for the `json_refs` storage
+     * decision, rather than calling `stringifyConsoleLogArgs` on the same
+     * `args` array twice.
+     */
+    argsJson: string | null;
+    location?: {
+        url?: string;
+        lineNumber?: number;
+        columnNumber?: number;
+    };
+    stack?: string;
+}
+/**
+ * The resolved ref ids and scalar fields that make up one
+ * `console_log_items` row, computed by the caller before the upsert runs.
+ */
+export interface ConsoleLogItemRow {
+    hash: Buffer;
+    type: string;
+    /** `null` when `text` is the empty string — `text_refs` never stores `''`. */
+    textId: number | null;
+    argsJsonId: number | null;
+    locUrlId: number | null;
+    locLine: number | null;
+    locColumn: number | null;
+    stackTextId: number | null;
+}

package/lib/archive/db-ops/console-logs/types.js

@@ -0,0 +1 @@
+export {};

package/lib/archive/db-ops/console-logs/upsert-console-log-item.d.ts

@@ -0,0 +1,29 @@
+import type { ConsoleLogItemRow } from './types.js';
+import type { WriteRefCaches } from '../_shared/types.js';
+import type { Knex } from 'knex';
+/**
+ * Resolves the `console_log_items.id` for one console log entry's content,
+ * inserting the dictionary row when the hash is not yet known.
+ *
+ * Same no-op-update-for-`RETURNING` idiom as `upsertUrlRef` / `upsertJsonRef`:
+ * `INSERT ... ON CONFLICT(hash) DO UPDATE SET hash = hash RETURNING id`
+ * yields the existing row's id on a hash collision without a separate
+ * SELECT round trip, and (unlike `INSERT OR IGNORE ... RETURNING`) still
+ * returns a row when the insert itself is the no-op.
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ * @param caches - The connection's write-side id caches; mutated in place.
+ * @param row - The content hash plus every resolved ref id / scalar column.
+ * @returns The `console_log_items.id` of the existing or newly inserted row.
+ * @example
+ * const id = await upsertConsoleLogItem(trx, caches, {
+ *   hash: computeConsoleLogHash(entry),
+ *   type: entry.type,
+ *   textId,
+ *   argsJsonId: null,
+ *   locUrlId: null,
+ *   locLine: null,
+ *   locColumn: null,
+ *   stackTextId: null,
+ * });
+ */
+export declare function upsertConsoleLogItem(qb: Knex | Knex.Transaction, caches: WriteRefCaches, row: ConsoleLogItemRow): Promise<number>;

package/lib/archive/db-ops/console-logs/upsert-console-log-item.js

@@ -0,0 +1,52 @@
+/**
+ * Resolves the `console_log_items.id` for one console log entry's content,
+ * inserting the dictionary row when the hash is not yet known.
+ *
+ * Same no-op-update-for-`RETURNING` idiom as `upsertUrlRef` / `upsertJsonRef`:
+ * `INSERT ... ON CONFLICT(hash) DO UPDATE SET hash = hash RETURNING id`
+ * yields the existing row's id on a hash collision without a separate
+ * SELECT round trip, and (unlike `INSERT OR IGNORE ... RETURNING`) still
+ * returns a row when the insert itself is the no-op.
+ * @param qb - Knex instance or transaction connected to the archive DB.
+ * @param caches - The connection's write-side id caches; mutated in place.
+ * @param row - The content hash plus every resolved ref id / scalar column.
+ * @returns The `console_log_items.id` of the existing or newly inserted row.
+ * @example
+ * const id = await upsertConsoleLogItem(trx, caches, {
+ *   hash: computeConsoleLogHash(entry),
+ *   type: entry.type,
+ *   textId,
+ *   argsJsonId: null,
+ *   locUrlId: null,
+ *   locLine: null,
+ *   locColumn: null,
+ *   stackTextId: null,
+ * });
+ */
+export async function upsertConsoleLogItem(qb, caches, row) {
+    const hex = row.hash.toString('hex');
+    const cached = caches.consoleLogIds.get(hex);
+    if (cached !== undefined) {
+        return cached;
+    }
+    const rows = await qb.raw(`INSERT INTO console_log_items
+			(hash, type, text_id, args_json_id, loc_url_id, loc_line, loc_column, stack_text_id)
+		 VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+		 ON CONFLICT(hash) DO UPDATE SET hash = hash
+		 RETURNING id`, [
+        row.hash,
+        row.type,
+        row.textId,
+        row.argsJsonId,
+        row.locUrlId,
+        row.locLine,
+        row.locColumn,
+        row.stackTextId,
+    ]);
+    const first = rows[0];
+    if (first === undefined) {
+        throw new Error('upsertConsoleLogItem: RETURNING yielded no row');
+    }
+    caches.consoleLogIds.set(hex, first.id);
+    return first.id;
+}

package/lib/archive/db-ops/dedupe-cap/accumulate-dedupe-cap-rejected-count.d.ts

@@ -0,0 +1,18 @@
+import type { Knex } from 'knex';
+/**
+ * Adds `rejectedCount` onto the `rejected_count` of the `dedupe_cap_events`
+ * row for `shapeKey`, treating a still-`NULL` count as `0`. Unlike
+ * `finalize-dedupe-cap-event.ts` (which stamps a session's own newly-capped
+ * shape exactly once, guarded by `whereNull`), this targets a shape that
+ * capped in an EARLIER session and was preloaded into `DedupeCapTracker`'s
+ * sticky set (see `DedupeCapTracker`'s constructor JSDoc) — gate rejections
+ * for such a shape still occur in the current session, but no `dedupeCap`
+ * event (and thus no new row) is ever emitted for it, since the tracker
+ * short-circuits on an already-sticky shape before `observe` runs. Matches
+ * by `shape_key` rather than `id` because the caller (`CrawlerOrchestrator`)
+ * only has the shape key for a preloaded-sticky shape, never its row id.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param shapeKey - The capped shape whose rejection count to accumulate.
+ * @param rejectedCount - Additional anchors rejected for this shape in the current session.
+ */
+export declare function accumulateDedupeCapRejectedCount(knex: Knex, shapeKey: string, rejectedCount: number): Promise<void>;

package/lib/archive/db-ops/dedupe-cap/accumulate-dedupe-cap-rejected-count.js

@@ -0,0 +1,23 @@
+/**
+ * Adds `rejectedCount` onto the `rejected_count` of the `dedupe_cap_events`
+ * row for `shapeKey`, treating a still-`NULL` count as `0`. Unlike
+ * `finalize-dedupe-cap-event.ts` (which stamps a session's own newly-capped
+ * shape exactly once, guarded by `whereNull`), this targets a shape that
+ * capped in an EARLIER session and was preloaded into `DedupeCapTracker`'s
+ * sticky set (see `DedupeCapTracker`'s constructor JSDoc) — gate rejections
+ * for such a shape still occur in the current session, but no `dedupeCap`
+ * event (and thus no new row) is ever emitted for it, since the tracker
+ * short-circuits on an already-sticky shape before `observe` runs. Matches
+ * by `shape_key` rather than `id` because the caller (`CrawlerOrchestrator`)
+ * only has the shape key for a preloaded-sticky shape, never its row id.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param shapeKey - The capped shape whose rejection count to accumulate.
+ * @param rejectedCount - Additional anchors rejected for this shape in the current session.
+ */
+export async function accumulateDedupeCapRejectedCount(knex, shapeKey, rejectedCount) {
+    await knex('dedupe_cap_events')
+        .where({ shape_key: shapeKey })
+        .update({
+        rejected_count: knex.raw('COALESCE(rejected_count, 0) + ?', [rejectedCount]),
+    });
+}

package/lib/archive/db-ops/dedupe-cap/finalize-dedupe-cap-event.d.ts

@@ -0,0 +1,12 @@
+import type { Knex } from 'knex';
+/**
+ * Finalizes a `dedupe_cap_events` row by stamping `rejected_count` — but
+ * ONLY if it is still unset. The `whereNull('rejected_count')` guard makes
+ * this idempotent, mirroring `close-network-outage.ts`'s `ended_at` guard: a
+ * second call matches zero rows and is a silent no-op rather than
+ * overwriting an already-finalized count.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param id - The `dedupe_cap_events.id` to finalize.
+ * @param rejectedCount - Number of anchors rejected for this shape after it capped.
+ */
+export declare function finalizeDedupeCapEvent(knex: Knex, id: number, rejectedCount: number): Promise<void>;

package/lib/archive/db-ops/dedupe-cap/finalize-dedupe-cap-event.js

@@ -0,0 +1,15 @@
+/**
+ * Finalizes a `dedupe_cap_events` row by stamping `rejected_count` — but
+ * ONLY if it is still unset. The `whereNull('rejected_count')` guard makes
+ * this idempotent, mirroring `close-network-outage.ts`'s `ended_at` guard: a
+ * second call matches zero rows and is a silent no-op rather than
+ * overwriting an already-finalized count.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param id - The `dedupe_cap_events.id` to finalize.
+ * @param rejectedCount - Number of anchors rejected for this shape after it capped.
+ */
+export async function finalizeDedupeCapEvent(knex, id, rejectedCount) {
+    await knex('dedupe_cap_events').where({ id }).whereNull('rejected_count').update({
+        rejected_count: rejectedCount,
+    });
+}

package/lib/archive/db-ops/dedupe-cap/insert-dedupe-cap-event.d.ts

@@ -0,0 +1,14 @@
+import type { InsertDedupeCapEventParams } from '../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Appends one row to the `dedupe_cap_events` journal, with `rejected_count`
+ * left `NULL` — the row starts life without a finalized rejection count.
+ *
+ * Called the instant `DedupeCapTracker#observe` confirms a URL shape as a
+ * same-cluster trap (the `dedupeCap` event). See
+ * `finalize-dedupe-cap-event.ts` for how `rejected_count` is later set.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param params - The newly-capped shape's fields to record.
+ * @returns The autoincremented `id` of the newly-inserted row.
+ */
+export declare function insertDedupeCapEvent(knex: Knex, params: InsertDedupeCapEventParams): Promise<number>;

package/lib/archive/db-ops/dedupe-cap/insert-dedupe-cap-event.js

@@ -0,0 +1,30 @@
+/**
+ * Appends one row to the `dedupe_cap_events` journal, with `rejected_count`
+ * left `NULL` — the row starts life without a finalized rejection count.
+ *
+ * Called the instant `DedupeCapTracker#observe` confirms a URL shape as a
+ * same-cluster trap (the `dedupeCap` event). See
+ * `finalize-dedupe-cap-event.ts` for how `rejected_count` is later set.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param params - The newly-capped shape's fields to record.
+ * @returns The autoincremented `id` of the newly-inserted row.
+ */
+export async function insertDedupeCapEvent(knex, params) {
+    const inserted = await knex
+        .from('dedupe_cap_events')
+        .insert({
+        shape_key: params.shapeKey,
+        sample_url: params.sampleUrl,
+        body_hash: params.bodyHash,
+        effective_threshold: params.effectiveThreshold,
+        observed_count: params.observedCount,
+        detected_at: params.detectedAt,
+        rejected_count: null,
+    })
+        .returning('id');
+    const id = inserted[0]?.id;
+    if (typeof id !== 'number') {
+        throw new TypeError('insertDedupeCapEvent: INSERT returned no row id');
+    }
+    return id;
+}

package/lib/archive/db-ops/dedupe-cap/list-dedupe-cap-shape-keys.d.ts

@@ -0,0 +1,21 @@
+import type { Knex } from 'knex';
+/**
+ * Every distinct `dedupe_cap_events.shape_key` recorded in this archive —
+ * used by `CrawlerOrchestrator` to preload `DedupeCapTracker`'s sticky set
+ * on `--resume` / `--append` / `--retry-failed` / `--inventory`, so a trap
+ * this crawl already paid the cost of discovering once is not re-admitted
+ * in a later session. Fresh (non-resuming) crawls do not call this — there
+ * is no archive history to seed from.
+ *
+ * Unlike `listDnsBurnedHostCandidates`, no additional exclusion logic is
+ * needed: once `DedupeCapTracker` confirms a shape as a trap, it stays
+ * confirmed — there is no equivalent of "the host might have recovered
+ * since".
+ *
+ * Returns `[]` on legacy archives that pre-date the `dedupe_cap_events`
+ * table (self-healed on next writer open, so this is never a permanent
+ * state) or that have recorded no capped shapes.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns Distinct shape keys already confirmed capped.
+ */
+export declare function listDedupeCapShapeKeys(knex: Knex): Promise<string[]>;

package/lib/archive/db-ops/dedupe-cap/list-dedupe-cap-shape-keys.js

@@ -0,0 +1,27 @@
+/**
+ * Every distinct `dedupe_cap_events.shape_key` recorded in this archive —
+ * used by `CrawlerOrchestrator` to preload `DedupeCapTracker`'s sticky set
+ * on `--resume` / `--append` / `--retry-failed` / `--inventory`, so a trap
+ * this crawl already paid the cost of discovering once is not re-admitted
+ * in a later session. Fresh (non-resuming) crawls do not call this — there
+ * is no archive history to seed from.
+ *
+ * Unlike `listDnsBurnedHostCandidates`, no additional exclusion logic is
+ * needed: once `DedupeCapTracker` confirms a shape as a trap, it stays
+ * confirmed — there is no equivalent of "the host might have recovered
+ * since".
+ *
+ * Returns `[]` on legacy archives that pre-date the `dedupe_cap_events`
+ * table (self-healed on next writer open, so this is never a permanent
+ * state) or that have recorded no capped shapes.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns Distinct shape keys already confirmed capped.
+ */
+export async function listDedupeCapShapeKeys(knex) {
+    const hasTable = await knex.schema.hasTable('dedupe_cap_events');
+    if (!hasTable) {
+        return [];
+    }
+    const rows = (await knex('dedupe_cap_events').distinct('shape_key'));
+    return rows.map((row) => row.shape_key);
+}

package/lib/archive/db-ops/errors/list-dns-burned-host-candidates.d.ts

@@ -16,6 +16,17 @@ import type { Knex } from 'knex';
  *
  * Returns `[]` on legacy archives that pre-date the `crawl_errors`
  * table — the `hasTable` guard keeps the call non-destructive.
+ *
+ * **Outage exclusion**: a host whose latest DNS error's `createdAt` falls
+ * inside a recorded `network_outages` window is dropped from the result
+ * regardless of the other checks. A host with no prior success this
+ * archive has ever seen (the case this whole exclusion-bag scheme cannot
+ * otherwise catch) would, without this check, get preload-seeded into
+ * `dnsBurnedHostCache` and short-circuit on EVERY subsequent session
+ * forever — the exact "damage 3" this function's caller
+ * (`#preloadDnsBurnedHostCache`) exists to prevent for hosts genuinely
+ * proven dead, not to inflict on hosts merely unlucky enough to be first
+ * contacted during an operator-side network blip.
  * @param knex - Knex query builder connected to the archive DB.
  * @returns Lower-cased hostnames safe to short-circuit.
  */

package/lib/archive/db-ops/errors/list-dns-burned-host-candidates.js

@@ -1,4 +1,6 @@
 import { classifyErrorKind } from '../../../classify-error-kind.js';
+import { isWithinOutageWindow } from '../../../is-within-outage-window.js';
+import { listNetworkOutages } from '../outages/list-network-outages.js';
 /**
  * Hostnames whose `crawl_errors` history is consistently DNS failures and
  * for which no recent 2xx-3xx page or resource is recorded — i.e. hosts
@@ -16,6 +18,17 @@ import { classifyErrorKind } from '../../../classify-error-kind.js';
  *
  * Returns `[]` on legacy archives that pre-date the `crawl_errors`
  * table — the `hasTable` guard keeps the call non-destructive.
+ *
+ * **Outage exclusion**: a host whose latest DNS error's `createdAt` falls
+ * inside a recorded `network_outages` window is dropped from the result
+ * regardless of the other checks. A host with no prior success this
+ * archive has ever seen (the case this whole exclusion-bag scheme cannot
+ * otherwise catch) would, without this check, get preload-seeded into
+ * `dnsBurnedHostCache` and short-circuit on EVERY subsequent session
+ * forever — the exact "damage 3" this function's caller
+ * (`#preloadDnsBurnedHostCache`) exists to prevent for hosts genuinely
+ * proven dead, not to inflict on hosts merely unlucky enough to be first
+ * contacted during an operator-side network blip.
  * @param knex - Knex query builder connected to the archive DB.
  * @returns Lower-cased hostnames safe to short-circuit.
  */
@@ -118,11 +131,12 @@ export async function listDnsBurnedHostCandidates(knex) {
         }
         resourceOkHosts.add(host);
     }
+    const outageWindows = await listNetworkOutages(knex);
     // A candidate host is burned only if neither pages nor resources hold a
-    // 2xx-3xx for it, AND its latest 2xx page (if any) is not newer than
-    // the latest DNS error. The third check guards against re-burning a
-    // host that recovered between the last DNS failure and the most recent
-    // crawl.
+    // 2xx-3xx for it, its latest 2xx page (if any) is not newer than the
+    // latest DNS error (guards against re-burning a host that recovered
+    // between the last DNS failure and the most recent crawl), AND that
+    // latest DNS error did not happen during a recorded network outage.
     const burned = [];
     for (const [host, latestErrorAt] of candidateLatestErrorAt) {
         if (pageOkHosts.has(host)) {
@@ -135,6 +149,9 @@ export async function listDnsBurnedHostCandidates(knex) {
         if (typeof latestOkAt === 'number' && latestOkAt > latestErrorAt) {
             continue;
         }
+        if (isWithinOutageWindow(latestErrorAt, outageWindows)) {
+            continue;
+        }
         burned.push(host);
     }
     return burned;

package/lib/archive/db-ops/inventory/record-inventory-run.js

@@ -27,6 +27,7 @@ export async function recordInventoryRun(knex, meta) {
         new_pages: meta.new_pages ?? null,
         new_resources: meta.new_resources ?? null,
         scope_skipped: meta.scope_skipped ?? null,
+        invalid_skipped: meta.invalid_skipped ?? null,
         notes: meta.notes ?? null,
     })
         .returning('id');

package/lib/archive/db-ops/lifecycle/init.d.ts

@@ -2,7 +2,7 @@ import type { Knex } from 'knex';
 /**
  * Initializes the database schema if tables do not exist, then runs the
  * remaining lightweight migrations (`info.roots`, `info.mainContentSelector`,
- * `page_meta.main_content_*`).
+ * `page_meta.main_content_*`, `inventory_runs.invalid_skipped`).
  *
  * There is deliberately no per-table *table-creation* migration chain here:
  * `assertCompatibleVersion` (called below, before any schema work) rejects
@@ -17,7 +17,16 @@ import type { Knex } from 'knex';
  * `CREATE TABLE IF NOT EXISTS` is a no-op once the table is present. Column
  * additions to an existing 0.13 table are therefore the one case that still
  * needs an explicit `hasColumn`-guarded `ALTER TABLE` here (`migrateInfoRoots`,
- * `migrateMainContentsColumns`) rather than a DDL-string change alone.
+ * `migrateMainContentsColumns`, `migratePageMetaBodyHash`,
+ * `migratePageMetaConsoleErrorCount`, `migrateContentItemsAliasOfId`,
+ * `migrateInventoryRunsInvalidSkipped`) rather than a DDL-string change alone.
+ *
+ * `closeStaleOpenNetworkOutages` is not a schema migration (no columns
+ * change) but belongs at this same boot phase for the same reason the
+ * others do: it must run before ANY reader (`resetFailedPages`,
+ * `listDnsBurnedHostCandidates`, …) can observe a `network_outages` row
+ * left `ended_at = NULL` by a crawl process that was killed mid-outage in a
+ * prior session.
  *
  * In read-only mode schema init + migration are SKIPPED so the same DB
  * can be opened safely by a viewer attached to a live (or interrupted)

package/lib/archive/db-ops/lifecycle/init.js

@@ -1,12 +1,17 @@
 import { applyConnectionPragmas, initSchema } from '../../init-schema.js';
 import { assertCompatibleVersion } from '../../meta/assert-compatible-version.js';
+import { migrateContentItemsAliasOfId } from '../../migrate-content-items-alias-of-id.js';
 import { migrateInfoMainContentSelector } from '../../migrate-info-main-content-selector.js';
 import { migrateInfoRoots } from '../../migrate-info-roots.js';
+import { migrateInventoryRunsInvalidSkipped } from '../../migrate-inventory-runs-invalid-skipped.js';
 import { migrateMainContentsColumns } from '../../migrate-main-contents-columns.js';
+import { migratePageMetaBodyHash } from '../../migrate-page-meta-body-hash.js';
+import { migratePageMetaConsoleErrorCount } from '../../migrate-page-meta-console-error-count.js';
+import { closeStaleOpenNetworkOutages } from '../outages/close-stale-open-network-outages.js';
 /**
  * Initializes the database schema if tables do not exist, then runs the
  * remaining lightweight migrations (`info.roots`, `info.mainContentSelector`,
- * `page_meta.main_content_*`).
+ * `page_meta.main_content_*`, `inventory_runs.invalid_skipped`).
  *
  * There is deliberately no per-table *table-creation* migration chain here:
  * `assertCompatibleVersion` (called below, before any schema work) rejects
@@ -21,7 +26,16 @@ import { migrateMainContentsColumns } from '../../migrate-main-contents-columns.
  * `CREATE TABLE IF NOT EXISTS` is a no-op once the table is present. Column
  * additions to an existing 0.13 table are therefore the one case that still
  * needs an explicit `hasColumn`-guarded `ALTER TABLE` here (`migrateInfoRoots`,
- * `migrateMainContentsColumns`) rather than a DDL-string change alone.
+ * `migrateMainContentsColumns`, `migratePageMetaBodyHash`,
+ * `migratePageMetaConsoleErrorCount`, `migrateContentItemsAliasOfId`,
+ * `migrateInventoryRunsInvalidSkipped`) rather than a DDL-string change alone.
+ *
+ * `closeStaleOpenNetworkOutages` is not a schema migration (no columns
+ * change) but belongs at this same boot phase for the same reason the
+ * others do: it must run before ANY reader (`resetFailedPages`,
+ * `listDnsBurnedHostCandidates`, …) can observe a `network_outages` row
+ * left `ended_at = NULL` by a crawl process that was killed mid-outage in a
+ * prior session.
  *
  * In read-only mode schema init + migration are SKIPPED so the same DB
  * can be opened safely by a viewer attached to a live (or interrupted)
@@ -49,4 +63,9 @@ export async function init(knex, readOnly) {
     await migrateInfoRoots(knex);
     await migrateInfoMainContentSelector(knex);
     await migrateMainContentsColumns(knex);
+    await migratePageMetaBodyHash(knex);
+    await migratePageMetaConsoleErrorCount(knex);
+    await migrateContentItemsAliasOfId(knex);
+    await migrateInventoryRunsInvalidSkipped(knex);
+    await closeStaleOpenNetworkOutages(knex);
 }

package/lib/archive/db-ops/outages/close-network-outage.d.ts

@@ -0,0 +1,12 @@
+import type { Knex } from 'knex';
+/**
+ * Close an outage row by stamping `ended_at` — but ONLY if it is still
+ * open. The `whereNull('ended_at')` guard is what makes this idempotent: a
+ * second call (e.g. a duplicate recovery-probe success racing the first)
+ * matches zero rows and is a silent no-op, rather than overwriting an
+ * already-recorded `ended_at` with a later timestamp.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param id - The `network_outages.id` to close.
+ * @param endedAt - Epoch ms the outage is considered to have ended.
+ */
+export declare function closeNetworkOutage(knex: Knex, id: number, endedAt: number): Promise<void>;

package/lib/archive/db-ops/outages/close-network-outage.js

@@ -0,0 +1,15 @@
+/**
+ * Close an outage row by stamping `ended_at` — but ONLY if it is still
+ * open. The `whereNull('ended_at')` guard is what makes this idempotent: a
+ * second call (e.g. a duplicate recovery-probe success racing the first)
+ * matches zero rows and is a silent no-op, rather than overwriting an
+ * already-recorded `ended_at` with a later timestamp.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param id - The `network_outages.id` to close.
+ * @param endedAt - Epoch ms the outage is considered to have ended.
+ */
+export async function closeNetworkOutage(knex, id, endedAt) {
+    await knex('network_outages').where({ id }).whereNull('ended_at').update({
+        ended_at: endedAt,
+    });
+}