@nitpicker/crawler 0.14.0 → 0.16.0

package/lib/archive/db-ops/outages/close-stale-open-network-outages.d.ts

@@ -0,0 +1,26 @@
+import type { Knex } from 'knex';
+/**
+ * Durably close every `network_outages` row still `ended_at = NULL` at the
+ * start of a writer session.
+ *
+ * A row is left open only when the crawl process was killed (Ctrl-C / OOM
+ * / SIGKILL) mid-outage, before a recovery probe could close it. Called
+ * once from `db-ops/lifecycle/init.ts` — right after `initSchema` (which
+ * guarantees the table exists) and before anything else touches
+ * `network_outages` — so that by the time `resetFailedPages` /
+ * `listDnsBurnedHostCandidates` / any other reader runs, no row can still
+ * be open from a PRIOR session. (A row opened by THIS session cannot exist
+ * yet at this point in the boot sequence — the sliding-window detector
+ * only starts once the crawl loop begins.)
+ *
+ * This complements, but does not replace, `list-network-outages.ts`'s
+ * on-the-fly clamp: that read-side resolution stays in place as a
+ * defensive fallback, but after this runs there should be nothing left for
+ * it to resolve.
+ *
+ * Idempotent: a row closed by a previous call (or by a normal
+ * recovery-probe success) is simply absent from the `whereNull('ended_at')`
+ * scan, so re-running this on every writer open is always safe.
+ * @param knex - Knex query builder connected to the archive DB.
+ */
+export declare function closeStaleOpenNetworkOutages(knex: Knex): Promise<void>;

package/lib/archive/db-ops/outages/close-stale-open-network-outages.js

@@ -0,0 +1,42 @@
+import { closeNetworkOutage } from './close-network-outage.js';
+import { computeOutageClampTimestamp } from './compute-outage-clamp-timestamp.js';
+/**
+ * Durably close every `network_outages` row still `ended_at = NULL` at the
+ * start of a writer session.
+ *
+ * A row is left open only when the crawl process was killed (Ctrl-C / OOM
+ * / SIGKILL) mid-outage, before a recovery probe could close it. Called
+ * once from `db-ops/lifecycle/init.ts` — right after `initSchema` (which
+ * guarantees the table exists) and before anything else touches
+ * `network_outages` — so that by the time `resetFailedPages` /
+ * `listDnsBurnedHostCandidates` / any other reader runs, no row can still
+ * be open from a PRIOR session. (A row opened by THIS session cannot exist
+ * yet at this point in the boot sequence — the sliding-window detector
+ * only starts once the crawl loop begins.)
+ *
+ * This complements, but does not replace, `list-network-outages.ts`'s
+ * on-the-fly clamp: that read-side resolution stays in place as a
+ * defensive fallback, but after this runs there should be nothing left for
+ * it to resolve.
+ *
+ * Idempotent: a row closed by a previous call (or by a normal
+ * recovery-probe success) is simply absent from the `whereNull('ended_at')`
+ * scan, so re-running this on every writer open is always safe.
+ * @param knex - Knex query builder connected to the archive DB.
+ */
+export async function closeStaleOpenNetworkOutages(knex) {
+    const hasTable = await knex.schema.hasTable('network_outages');
+    if (!hasTable) {
+        return;
+    }
+    const openRows = (await knex('network_outages')
+        .whereNull('ended_at')
+        .select('id', 'started_at'));
+    if (openRows.length === 0) {
+        return;
+    }
+    const clamp = await computeOutageClampTimestamp(knex);
+    for (const row of openRows) {
+        await closeNetworkOutage(knex, row.id, Math.max(clamp, row.started_at));
+    }
+}

package/lib/archive/db-ops/outages/compute-outage-clamp-timestamp.d.ts

@@ -0,0 +1,26 @@
+import type { Knex } from 'knex';
+/**
+ * Compute the timestamp used to resolve an outage row whose `ended_at` is
+ * still `NULL` (the crawl process was killed before a recovery probe could
+ * close it) into a concrete, bounded window.
+ *
+ * The archive's own most-recent observations — the latest `crawl_errors`
+ * timestamp and the latest `content_items.last_crawled_at` — are the only
+ * evidence available for "when did activity in this archive last happen",
+ * so the later of the two stands in for "the outage cannot have lasted
+ * past this point, because the archive shows activity here". Using the
+ * LARGER of the two (not just `crawl_errors`) matters because a session
+ * that crashed mid-outage may have its last successful page write be more
+ * recent than its last recorded error, or vice versa.
+ *
+ * Both source tables are guaranteed to exist by the time this runs — this
+ * helper is only ever called from writer-context code (see
+ * `list-network-outages.ts` / the boot-time stale-open finalizer), and
+ * `initSchema` creates both `crawl_errors` and `content_items` before any
+ * writer session's first query.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The larger of `MAX(crawl_errors.createdAt)` and
+ *   `MAX(content_items.last_crawled_at)`, or `0` if the archive has neither
+ *   (a brand-new archive with no activity yet).
+ */
+export declare function computeOutageClampTimestamp(knex: Knex): Promise<number>;

package/lib/archive/db-ops/outages/compute-outage-clamp-timestamp.js

@@ -0,0 +1,31 @@
+/**
+ * Compute the timestamp used to resolve an outage row whose `ended_at` is
+ * still `NULL` (the crawl process was killed before a recovery probe could
+ * close it) into a concrete, bounded window.
+ *
+ * The archive's own most-recent observations — the latest `crawl_errors`
+ * timestamp and the latest `content_items.last_crawled_at` — are the only
+ * evidence available for "when did activity in this archive last happen",
+ * so the later of the two stands in for "the outage cannot have lasted
+ * past this point, because the archive shows activity here". Using the
+ * LARGER of the two (not just `crawl_errors`) matters because a session
+ * that crashed mid-outage may have its last successful page write be more
+ * recent than its last recorded error, or vice versa.
+ *
+ * Both source tables are guaranteed to exist by the time this runs — this
+ * helper is only ever called from writer-context code (see
+ * `list-network-outages.ts` / the boot-time stale-open finalizer), and
+ * `initSchema` creates both `crawl_errors` and `content_items` before any
+ * writer session's first query.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The larger of `MAX(crawl_errors.createdAt)` and
+ *   `MAX(content_items.last_crawled_at)`, or `0` if the archive has neither
+ *   (a brand-new archive with no activity yet).
+ */
+export async function computeOutageClampTimestamp(knex) {
+    const [crawlErrorsRow] = (await knex('crawl_errors').max('createdAt as maxCreatedAt'));
+    const [contentItemsRow] = (await knex('content_items').max('last_crawled_at as maxLastCrawledAt'));
+    const latestError = crawlErrorsRow?.maxCreatedAt ?? 0;
+    const latestCrawl = contentItemsRow?.maxLastCrawledAt ?? 0;
+    return Math.max(latestError, latestCrawl);
+}

package/lib/archive/db-ops/outages/insert-network-outage.d.ts

@@ -0,0 +1,14 @@
+import type { InsertNetworkOutageParams } from '../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Appends one row to the `network_outages` journal, with `ended_at` left
+ * `NULL` — the row starts life as an open outage.
+ *
+ * Called the moment a recovery probe CONFIRMS a suspect outage (i.e. the
+ * probe itself failed, not merely the sliding-window threshold trip). See
+ * `close-network-outage.ts` for how the row is later closed.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param params - The confirmed-outage fields to record.
+ * @returns The autoincremented `id` of the newly-inserted row.
+ */
+export declare function insertNetworkOutage(knex: Knex, params: InsertNetworkOutageParams): Promise<number>;

package/lib/archive/db-ops/outages/insert-network-outage.js

@@ -0,0 +1,29 @@
+/**
+ * Appends one row to the `network_outages` journal, with `ended_at` left
+ * `NULL` — the row starts life as an open outage.
+ *
+ * Called the moment a recovery probe CONFIRMS a suspect outage (i.e. the
+ * probe itself failed, not merely the sliding-window threshold trip). See
+ * `close-network-outage.ts` for how the row is later closed.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param params - The confirmed-outage fields to record.
+ * @returns The autoincremented `id` of the newly-inserted row.
+ */
+export async function insertNetworkOutage(knex, params) {
+    const inserted = await knex
+        .from('network_outages')
+        .insert({
+        started_at: params.startedAt,
+        detected_at: params.detectedAt,
+        ended_at: null,
+        probe_host: params.probeHost,
+        trigger_error_count: params.triggerErrorCount,
+        trigger_host_count: params.triggerHostCount,
+    })
+        .returning('id');
+    const id = inserted[0]?.id;
+    if (typeof id !== 'number') {
+        throw new TypeError('insertNetworkOutage: INSERT returned no row id');
+    }
+    return id;
+}

package/lib/archive/db-ops/outages/list-network-outages.d.ts

@@ -0,0 +1,28 @@
+import type { OutageWindow } from '../../../is-within-outage-window.js';
+import type { Knex } from 'knex';
+/**
+ * List every recorded outage as a resolved (closed) {@link OutageWindow},
+ * suitable for `isWithinOutageWindow`.
+ *
+ * This is the crawler-internal counterpart to `@nitpicker/query`'s richer
+ * `listNetworkOutages` (full row shape, for CLI/MCP display) — this one
+ * exists purely to feed the three write-path consumers
+ * (`resetFailedPages`, `listDnsBurnedHostCandidates`, the gate itself) that
+ * only need "was this timestamp inside an outage", never the row's other
+ * columns.
+ *
+ * Any row whose `ended_at` is still `NULL` (the crawl process was killed
+ * before recovery) is resolved on the fly via
+ * {@link computeOutageClampTimestamp} rather than ever being returned as an
+ * unbounded window — see `is-within-outage-window.ts`'s `OutageWindow`
+ * docstring for why an open-ended window would be a correctness bug (every
+ * later error would retroactively read as network-caused, forever). This
+ * on-the-fly resolution does NOT persist to the row — it is a defensive
+ * fallback independent of whichever boot-time finalizer durably closes
+ * stale-open rows.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns Resolved outage windows. Empty when the archive predates the
+ *   `network_outages` table (self-healed on next writer open, so this is
+ *   never a permanent state) or has recorded no outages.
+ */
+export declare function listNetworkOutages(knex: Knex): Promise<OutageWindow[]>;

package/lib/archive/db-ops/outages/list-network-outages.js

@@ -0,0 +1,42 @@
+import { computeOutageClampTimestamp } from './compute-outage-clamp-timestamp.js';
+/**
+ * List every recorded outage as a resolved (closed) {@link OutageWindow},
+ * suitable for `isWithinOutageWindow`.
+ *
+ * This is the crawler-internal counterpart to `@nitpicker/query`'s richer
+ * `listNetworkOutages` (full row shape, for CLI/MCP display) — this one
+ * exists purely to feed the three write-path consumers
+ * (`resetFailedPages`, `listDnsBurnedHostCandidates`, the gate itself) that
+ * only need "was this timestamp inside an outage", never the row's other
+ * columns.
+ *
+ * Any row whose `ended_at` is still `NULL` (the crawl process was killed
+ * before recovery) is resolved on the fly via
+ * {@link computeOutageClampTimestamp} rather than ever being returned as an
+ * unbounded window — see `is-within-outage-window.ts`'s `OutageWindow`
+ * docstring for why an open-ended window would be a correctness bug (every
+ * later error would retroactively read as network-caused, forever). This
+ * on-the-fly resolution does NOT persist to the row — it is a defensive
+ * fallback independent of whichever boot-time finalizer durably closes
+ * stale-open rows.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns Resolved outage windows. Empty when the archive predates the
+ *   `network_outages` table (self-healed on next writer open, so this is
+ *   never a permanent state) or has recorded no outages.
+ */
+export async function listNetworkOutages(knex) {
+    const hasTable = await knex.schema.hasTable('network_outages');
+    if (!hasTable) {
+        return [];
+    }
+    const rows = (await knex('network_outages').select('started_at', 'ended_at'));
+    if (rows.length === 0) {
+        return [];
+    }
+    const hasOpenRow = rows.some((row) => row.ended_at === null);
+    const clamp = hasOpenRow ? await computeOutageClampTimestamp(knex) : 0;
+    return rows.map((row) => ({
+        startedAt: row.started_at,
+        endedAt: row.ended_at ?? Math.max(clamp, row.started_at),
+    }));
+}

package/lib/archive/db-ops/pages/reset/reset-failed-pages.d.ts

@@ -29,6 +29,19 @@ import type { Knex } from 'knex';
  * target shrinking across `--retry-failed` passes by leaving deterministic
  * dead-ends alone.
  *
+ * **Outage override**: before applying the permanent-kind exclusion, the
+ * message's `createdAt` is checked against every recorded
+ * `network_outages` window (see `is-within-outage-window.ts`). A `dns` (or
+ * any other permanent-kind) failure whose timestamp falls inside a window
+ * is treated as retryable regardless — `dns` is only a permanent,
+ * site-specific verdict when nothing else explains it; inside a confirmed
+ * operator-network outage, the same `getaddrinfo ENOTFOUND` message is
+ * evidence about the CRAWLER's connectivity, not the target site, and
+ * excluding it from retry would strand a perfectly reachable host as a
+ * false permanent failure for the rest of the archive's life. An archive
+ * with no recorded outages (`listNetworkOutages` returns `[]`) behaves
+ * exactly as before this override existed.
+ *
  * Matching rows — internal and external alike — are demoted back to pending
  * (`scraped = 0`) and have their stale scrape metadata cleared (the
  * `page_meta` row is deleted outright rather than nulled column-by-column).

package/lib/archive/db-ops/pages/reset/reset-failed-pages.js

@@ -1,7 +1,9 @@
 import { classifyErrorKind } from '../../../../classify-error-kind.js';
+import { isWithinOutageWindow } from '../../../../is-within-outage-window.js';
 import { PERMANENT_ERROR_KINDS } from '../../../../permanent-error-kinds.js';
 import { dbLog } from '../../../debug.js';
 import { getFailedPageMessages } from '../../../get-failed-page-messages.js';
+import { listNetworkOutages } from '../../outages/list-network-outages.js';
 /**
  * Reset previously-attempted pages that ended in a recoverable failure so a
  * follow-up crawl can re-fetch them from scratch.
@@ -32,6 +34,19 @@ import { getFailedPageMessages } from '../../../get-failed-page-messages.js';
  * target shrinking across `--retry-failed` passes by leaving deterministic
  * dead-ends alone.
  *
+ * **Outage override**: before applying the permanent-kind exclusion, the
+ * message's `createdAt` is checked against every recorded
+ * `network_outages` window (see `is-within-outage-window.ts`). A `dns` (or
+ * any other permanent-kind) failure whose timestamp falls inside a window
+ * is treated as retryable regardless — `dns` is only a permanent,
+ * site-specific verdict when nothing else explains it; inside a confirmed
+ * operator-network outage, the same `getaddrinfo ENOTFOUND` message is
+ * evidence about the CRAWLER's connectivity, not the target site, and
+ * excluding it from retry would strand a perfectly reachable host as a
+ * false permanent failure for the rest of the archive's life. An archive
+ * with no recorded outages (`listNetworkOutages` returns `[]`) behaves
+ * exactly as before this override existed.
+ *
  * Matching rows — internal and external alike — are demoted back to pending
  * (`scraped = 0`) and have their stale scrape metadata cleared (the
  * `page_meta` row is deleted outright rather than nulled column-by-column).
@@ -69,16 +84,29 @@ export async function resetFailedPages(knex) {
     }
     const candidateIds = candidates.map((row) => row.id);
     const candidateUrls = candidates.map((row) => row.url);
-    const messages = await getFailedPageMessages(knex, candidateIds, candidateUrls);
-    // Drop candidates whose latest recorded message classifies as permanent.
-    // An empty/absent message stays in the retry pool — we keep retrying when
-    // we don't know it's permanent, erring on the side of investigation.
+    // Unrelated tables (page_errors/crawl_errors vs network_outages), no data
+    // dependency between them — run concurrently instead of paying two
+    // sequential round-trips on every `--retry-failed` pass.
+    const [messages, outageWindows] = await Promise.all([
+        getFailedPageMessages(knex, candidateIds, candidateUrls),
+        listNetworkOutages(knex),
+    ]);
+    // Drop candidates whose latest recorded message classifies as permanent —
+    // UNLESS that message's timestamp falls inside a recorded network outage,
+    // in which case the permanent-kind verdict is overridden (see the
+    // "Outage override" section of this function's docstring). An
+    // empty/absent message stays in the retry pool regardless — we keep
+    // retrying when we don't know it's permanent, erring on the side of
+    // investigation.
     const retryable = candidates.filter((row) => {
-        const message = messages.get(row.id) ?? '';
-        if (message === '') {
+        const resolved = messages.get(row.id);
+        if (resolved === undefined || resolved.message === '') {
+            return true;
+        }
+        if (!PERMANENT_ERROR_KINDS.has(classifyErrorKind(resolved.message))) {
             return true;
         }
-        return !PERMANENT_ERROR_KINDS.has(classifyErrorKind(message));
+        return isWithinOutageWindow(resolved.createdAt, outageWindows);
     });
     const excludedCount = candidates.length - retryable.length;
     if (excludedCount > 0) {

package/lib/archive/db-ops/pages/write/update-page.js

@@ -1,6 +1,7 @@
 import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
 import { isHtmlContentType } from '../../../../crawler/is-html-content-type.js';
 import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+import { computeBodyHash } from '../../../body-hash/compute-body-hash.js';
 import { dbLog } from '../../../debug.js';
 import { deriveLineageFromParent } from '../../../derive-lineage-from-parent.js';
 import { matchImagesToDomPaths } from '../../../populate-entity-tables/match-images-to-dom-paths.js';
@@ -141,6 +142,12 @@ async function updatePageInTransaction(trx, knex, caches, page, destUrlObject, s
     // content check alone expresses the intent without a redundant term.
     if (writeHtml && page.html.length > 0) {
         await writePageHtmlBlob(pageId, page.html, trx);
+        // Computed from the same `page.html` written above, in the same
+        // transaction, so `page_meta.body_hash` never observably lags the
+        // snapshot it was derived from.
+        await trx('page_meta')
+            .where('page_id', pageId)
+            .update({ body_hash: computeBodyHash(page.html) });
     }
     else if (writeHtml &&
         page.contentType !== null &&
@@ -155,7 +162,12 @@ async function updatePageInTransaction(trx, knex, caches, page, destUrlObject, s
         // stale ref can only have been written by a snapshot-capable call
         // (`setPage`); `setExternalPage` passes `writeHtml = false` and never
         // sets `html`, so it has nothing to clear.
+        //
+        // `page_meta.body_hash` is cleared alongside it for the same reason —
+        // it is derived from the same HTML `page_html_ref` points at, so it
+        // must not outlive the ref it was computed from.
         await trx('page_html_ref').where('page_id', pageId).delete();
+        await trx('page_meta').where('page_id', pageId).update({ body_hash: null });
     }
     // Re-scrape semantics: the same URL can be scraped more than once
     // (e.g. `crawl --resume`, re-visits, `--append` re-promotion). Edges

package/lib/archive/db-ops/pages/write/write-page-html-blob.js

@@ -1,5 +1,5 @@
 import { createHash } from 'node:crypto';
-import { zstdCompressSync } from 'node:zlib';
+import { compressPayload } from '../../_shared/compress-payload.js';
 /**
  * Encodes, dedups, and persists a page's HTML snapshot.
  *
@@ -20,14 +20,14 @@ import { zstdCompressSync } from 'node:zlib';
 export async function writePageHtmlBlob(pageId, html, trx) {
     const rawBytes = Buffer.from(html, 'utf8');
     const hash = createHash('sha256').update(rawBytes).digest();
-    const compressed = zstdCompressSync(rawBytes);
+    const { body, codec, sizeRaw, sizeStored } = compressPayload(rawBytes);
     await trx('page_html_blobs')
         .insert({
         hash,
-        body: compressed,
-        codec: 'zstd',
-        size_raw: rawBytes.byteLength,
-        size_stored: compressed.byteLength,
+        body,
+        codec,
+        size_raw: sizeRaw,
+        size_stored: sizeStored,
     })
         .onConflict('hash')
         .ignore();

package/lib/archive/filesystem/output-binary.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Writes raw bytes to a file at the specified path, creating parent
+ * directories if needed.
+ *
+ * Unlike {@link outputText}, the buffer is written verbatim with no UTF-8
+ * re-encoding, so callers that need byte-for-byte fidelity (e.g. archiving
+ * a source file of unknown or mixed encoding for audit purposes) are not
+ * exposed to lossy round-tripping through a JS string.
+ * @param filePath - The absolute or relative path to the file to write.
+ * @param data - The raw bytes to write.
+ */
+export declare function outputBinary(filePath: string, data: Buffer): Promise<void>;

package/lib/archive/filesystem/output-binary.js

@@ -0,0 +1,17 @@
+import { promises as fs } from 'node:fs';
+import { mkdir } from './mkdir.js';
+/**
+ * Writes raw bytes to a file at the specified path, creating parent
+ * directories if needed.
+ *
+ * Unlike {@link outputText}, the buffer is written verbatim with no UTF-8
+ * re-encoding, so callers that need byte-for-byte fidelity (e.g. archiving
+ * a source file of unknown or mixed encoding for audit purposes) are not
+ * exposed to lossy round-tripping through a JS string.
+ * @param filePath - The absolute or relative path to the file to write.
+ * @param data - The raw bytes to write.
+ */
+export async function outputBinary(filePath, data) {
+    mkdir(filePath);
+    await fs.writeFile(filePath, data);
+}

package/lib/archive/get-failed-page-messages.d.ts

@@ -1,10 +1,22 @@
 import type { Knex } from 'knex';
 /**
- * Bulk-resolve a raw error message for each given page id, using only sources
- * reachable from a {@link Knex} handle. Read order: `page_errors` (keyed by
- * `pageId`, the most direct signal a scrape attempt recorded), then
- * `crawl_errors` (keyed by `url`, the crawler-channel record for failures
- * that happened before a page row was scraped).
+ * One resolved failure message, with the timestamp it was recorded at.
+ * `createdAt` is what lets a caller (`resetFailedPages`) decide whether the
+ * failure falls inside a `network_outages` window and should be treated as
+ * retryable regardless of its classified `ErrorKind` — see
+ * `is-within-outage-window.ts`.
+ */
+export interface FailedPageMessage {
+    message: string;
+    /** Epoch ms the message was recorded (`page_errors.createdAt` or `crawl_errors.createdAt`). */
+    createdAt: number;
+}
+/**
+ * Bulk-resolve a raw error message (plus its timestamp) for each given page
+ * id, using only sources reachable from a {@link Knex} handle. Read order:
+ * `page_errors` (keyed by `pageId`, the most direct signal a scrape attempt
+ * recorded), then `crawl_errors` (keyed by `url`, the crawler-channel
+ * record for failures that happened before a page row was scraped).
  *
  * **Known limitation — pre-`crawl_errors` archives**: This helper does NOT
  * read `error.log`. The `crawl_errors` table is created empty (by
@@ -30,8 +42,8 @@ import type { Knex } from 'knex';
  * @param urls - The corresponding `pages.url` values, in the same order as
  *   `ids`. Length and indexing MUST match `ids` so the page → url join can be
  *   reconstructed without a second `pages` round-trip.
- * @returns `Map<pageId, message>` populated only for ids whose message was
- *   found in one of the consulted tables.
+ * @returns `Map<pageId, FailedPageMessage>` populated only for ids whose
+ *   message was found in one of the consulted tables.
  * @example
  * ```ts
  * const messages = await getFailedPageMessages(
@@ -41,4 +53,4 @@ import type { Knex } from 'knex';
  * );
  * ```
  */
-export declare function getFailedPageMessages(instance: Knex, ids: readonly number[], urls: readonly string[]): Promise<Map<number, string>>;
+export declare function getFailedPageMessages(instance: Knex, ids: readonly number[], urls: readonly string[]): Promise<Map<number, FailedPageMessage>>;

package/lib/archive/get-failed-page-messages.js

@@ -1,11 +1,11 @@
 /** chunk size for `WHERE … IN (?)` SQLite parameter packing. */
 const CHUNK_SIZE = 500;
 /**
- * Bulk-resolve a raw error message for each given page id, using only sources
- * reachable from a {@link Knex} handle. Read order: `page_errors` (keyed by
- * `pageId`, the most direct signal a scrape attempt recorded), then
- * `crawl_errors` (keyed by `url`, the crawler-channel record for failures
- * that happened before a page row was scraped).
+ * Bulk-resolve a raw error message (plus its timestamp) for each given page
+ * id, using only sources reachable from a {@link Knex} handle. Read order:
+ * `page_errors` (keyed by `pageId`, the most direct signal a scrape attempt
+ * recorded), then `crawl_errors` (keyed by `url`, the crawler-channel
+ * record for failures that happened before a page row was scraped).
  *
  * **Known limitation — pre-`crawl_errors` archives**: This helper does NOT
  * read `error.log`. The `crawl_errors` table is created empty (by
@@ -31,8 +31,8 @@ const CHUNK_SIZE = 500;
  * @param urls - The corresponding `pages.url` values, in the same order as
  *   `ids`. Length and indexing MUST match `ids` so the page → url join can be
  *   reconstructed without a second `pages` round-trip.
- * @returns `Map<pageId, message>` populated only for ids whose message was
- *   found in one of the consulted tables.
+ * @returns `Map<pageId, FailedPageMessage>` populated only for ids whose
+ *   message was found in one of the consulted tables.
  * @example
  * ```ts
  * const messages = await getFailedPageMessages(
@@ -61,7 +61,7 @@ export async function getFailedPageMessages(instance, ids, urls) {
             // it as `parse-error` on the next run when the rows happen to be
             // returned in a different order.
             const rows = (await instance('page_errors')
-                .select('pageId', 'message')
+                .select('pageId', 'message', 'createdAt')
                 .whereIn('pageId', chunk)
                 .orderBy('id', 'asc'));
             for (const row of rows) {
@@ -79,7 +79,10 @@ export async function getFailedPageMessages(instance, ids, urls) {
                 // `dns` / `tls` / `client-blocked` etc. — defeating
                 // `--retry-failed`'s permanent-kind exclusion.
                 if (row.message !== '' && !messageByPageId.has(row.pageId)) {
-                    messageByPageId.set(row.pageId, row.message);
+                    messageByPageId.set(row.pageId, {
+                        message: row.message,
+                        createdAt: row.createdAt,
+                    });
                 }
             }
         }
@@ -110,11 +113,22 @@ export async function getFailedPageMessages(instance, ids, urls) {
     for (let i = 0; i < missingUrls.length; i += CHUNK_SIZE) {
         const chunk = missingUrls.slice(i, i + CHUNK_SIZE);
         const rows = (await instance('crawl_errors')
-            .select('url', 'message')
+            .select('url', 'message', 'createdAt')
             .whereIn('url', chunk));
         for (const row of rows) {
-            if (row.url !== null && !urlToMessage.has(row.url)) {
-                urlToMessage.set(row.url, row.message);
+            if (row.url === null) {
+                continue;
+            }
+            // Latest-createdAt wins (fixes a previously-undefined
+            // selection among duplicate URLs — SQLite's natural scan
+            // order is implementation-defined). The most recent message
+            // is the most relevant one for both classification and outage
+            // attribution: an old NXDOMAIN followed by a network-outage
+            // blip should resolve to the outage-era message, not whichever
+            // happened to be inserted first.
+            const existing = urlToMessage.get(row.url);
+            if (existing === undefined || row.createdAt > existing.createdAt) {
+                urlToMessage.set(row.url, { message: row.message, createdAt: row.createdAt });
             }
         }
     }
@@ -123,9 +137,9 @@ export async function getFailedPageMessages(instance, ids, urls) {
         if (url === undefined) {
             continue;
         }
-        const message = urlToMessage.get(url);
-        if (message !== undefined) {
-            messageByPageId.set(id, message);
+        const resolved = urlToMessage.get(url);
+        if (resolved !== undefined) {
+            messageByPageId.set(id, resolved);
         }
     }
     return messageByPageId;

package/lib/archive/migrate-content-items-alias-of-id.d.ts

@@ -0,0 +1,47 @@
+import type { Knex } from 'knex';
+/**
+ * Adds the `content_items.alias_of_id` column to archives created before
+ * this feature, then ensures its index exists.
+ *
+ * `content_items` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `content_items` predates this change. This
+ * mirrors {@link import('./migrate-page-meta-body-hash.js').migratePageMetaBodyHash}'s
+ * catch-up: a `hasColumn`-guarded `ALTER TABLE` for the one column
+ * `CREATE TABLE IF NOT EXISTS` cannot retrofit.
+ *
+ * Uses a raw `ALTER TABLE` (not the knex schema builder) so the retrofitted
+ * column's `REFERENCES content_items(id) DEFERRABLE INITIALLY DEFERRED`
+ * constraint matches the fresh-archive DDL bit-for-bit — the same
+ * self-referencing shape as `redirect_dest_id` (see
+ * `create-entity-tables.ts`'s JSDoc on why that FK must be deferred: a
+ * lower-id row can reference a higher-id row within the same write).
+ *
+ * The index is created here — unconditionally, after the column-add guard,
+ * not inside `createEntityTables`'s DDL — for both a fresh archive (where
+ * `alias_of_id` already exists from the DDL, so only the index still needs
+ * creating) and a legacy archive (where the column is added just above,
+ * then the index follows in the same call). `createEntityTables` runs
+ * unconditionally on every archive open, including legacy archives that
+ * still lack `alias_of_id` at that point; an unconditional
+ * `CREATE INDEX ... alias_of_id` there would fail with `no such column`
+ * before this migration ever runs. This function is the one place
+ * guaranteed to run only after the column is confirmed present, for both
+ * archive kinds.
+ *
+ * Only adds the column and its index — it does not compute values for
+ * existing rows (they stay `NULL`). That computation runs separately, from
+ * `backfillAliasOfId` during a viewer-read-model build, since it requires
+ * comparing every page's title and (for the trailing-slash tier) its
+ * `body_hash` against every other page.
+ *
+ * Idempotent: adding the column is a no-op once it exists (the index
+ * creation always runs, but `IF NOT EXISTS` makes repeat runs a no-op too).
+ * Guards on `content_items`'s existence defensively, though by the time this
+ * runs (after `initSchema`, itself after `assertCompatibleVersion` rejects
+ * pre-0.13 archives) the table is always present.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function migrateContentItemsAliasOfId(instance: Knex): Promise<void>;