@nitpicker/crawler 0.11.0 → 0.13.0

package/lib/archive/verify-migration/check-resource-items-count.js

@@ -0,0 +1,33 @@
+import { MigrationVerificationError } from './types.js';
+/**
+ * Verifies 0.13 invariant #6: every legacy `resources` row is mirrored
+ * by one row in `resource_items`.
+ *
+ * `populateResourceItems` fills `resource_items` with the same PK as
+ * `resources.id`
+ * (see `populate-resource-items.ts`); the invariant is broken only if the
+ * populate loop skipped rows during URL / header-set / content-type
+ * resolution. The paired invariant `resource_ref_edges = "resources-referrers"`
+ * is left unchecked here because issue #194's spec list stops at the
+ * six enumerated checks, and `resource_ref_edges` populate is a straight
+ * `INSERT … SELECT` (no dedup, no resolution) with negligible failure surface.
+ * @param trx - Knex instance or transaction connected to the populated archive.
+ * @throws {MigrationVerificationError} when the row counts diverge.
+ */
+export async function checkResourceItemsCount(trx) {
+    const resourceItemsRows = await trx('resource_items').count({
+        n: '*',
+    });
+    const resourcesRows = await trx('resources').count({ n: '*' });
+    const resourceItemsCount = Number(resourceItemsRows[0].n);
+    const resourcesCount = Number(resourcesRows[0].n);
+    if (resourceItemsCount !== resourcesCount) {
+        throw new MigrationVerificationError({
+            check: '#6 resource_items row count',
+            context: {
+                resource_items: resourceItemsCount,
+                resources: resourcesCount,
+            },
+        });
+    }
+}

package/lib/archive/verify-migration/check-url-round-trip.d.ts

@@ -0,0 +1,43 @@
+import type { Knex } from 'knex';
+/**
+ * Verifies 0.13 invariant #8: `content_items.url_id → url_refs.url`
+ * round-trips back to the original `pages.url` on a deterministic
+ * ≤ {@link SAMPLE_SIZE} sample.
+ *
+ * A mismatch means one of two things:
+ *
+ * - `populate-url-refs.ts` collapsed two distinct URLs into the same
+ *   `url_refs.id` (BINARY collation bug, whitespace normalisation, …).
+ * - `populate-content-items.ts` associated a page id with the wrong
+ *   `url_ref.id` (row-order dependency in the batch resolver).
+ *
+ * Additionally an FK gap (`content_items.url_id` pointing at a
+ * non-existent `url_refs.id`) or a missing `pages` row surfaces as a
+ * `roundTripUrl` / `sourceUrl` of `null`; the check treats either as a
+ * round-trip failure. LEFT JOINs are used deliberately so orphan rows
+ * are observable — an INNER JOIN would silently drop them and let the
+ * bug the invariant is meant to catch slip through.
+ *
+ * Sampling is **deterministic**: stride = ⌈count(content_items) / N⌉
+ * gives approximately {@link SAMPLE_SIZE} rows spread across the id
+ * range for large archives, and the full table for archives with
+ * ≤ {@link SAMPLE_SIZE} rows. Deterministic sampling means the same
+ * archive always produces the same verdict; a random sample would let a
+ * failing archive pass on retry if the offending rows happen to fall
+ * outside the second draw.
+ *
+ * The check throws when the sampled row count is smaller than expected
+ * (LEFT JOINs may return `null` sides but never fewer rows than the
+ * driving `content_items` set — a shortfall means SQLite silently
+ * dropped rows we intended to inspect). "Expected" is a separate
+ * `COUNT(*)` against the same `(id % stride) = 0` predicate, not a
+ * `floor(total / stride)` estimate: `content_items.id` is copied
+ * verbatim from the legacy `pages.id`, which accumulates gaps over an
+ * archive's crawl history (redirect consolidation, deleted pages, …).
+ * A density estimate over-counts whenever the id range is sparser than
+ * `[1, total]`, which produced false-positive failures on real
+ * multi-thousand-page archives.
+ * @param trx - Knex instance or transaction connected to the populated archive.
+ * @throws {MigrationVerificationError} when at least one sampled URL does not round-trip.
+ */
+export declare function checkUrlRoundTrip(trx: Knex): Promise<void>;

package/lib/archive/verify-migration/check-url-round-trip.js

@@ -0,0 +1,112 @@
+import { MigrationVerificationError } from './types.js';
+/**
+ * Sample size for the URL round-trip smoke test — a 1,000-row sample per
+ * issue #194.
+ * Not statistical coverage — URL normalisation bugs tend to be systematic
+ * (all rows with a particular scheme or host), so a modest evenly-spread
+ * sample catches them reliably.
+ */
+const SAMPLE_SIZE = 1000;
+/**
+ * Verifies 0.13 invariant #8: `content_items.url_id → url_refs.url`
+ * round-trips back to the original `pages.url` on a deterministic
+ * ≤ {@link SAMPLE_SIZE} sample.
+ *
+ * A mismatch means one of two things:
+ *
+ * - `populate-url-refs.ts` collapsed two distinct URLs into the same
+ *   `url_refs.id` (BINARY collation bug, whitespace normalisation, …).
+ * - `populate-content-items.ts` associated a page id with the wrong
+ *   `url_ref.id` (row-order dependency in the batch resolver).
+ *
+ * Additionally an FK gap (`content_items.url_id` pointing at a
+ * non-existent `url_refs.id`) or a missing `pages` row surfaces as a
+ * `roundTripUrl` / `sourceUrl` of `null`; the check treats either as a
+ * round-trip failure. LEFT JOINs are used deliberately so orphan rows
+ * are observable — an INNER JOIN would silently drop them and let the
+ * bug the invariant is meant to catch slip through.
+ *
+ * Sampling is **deterministic**: stride = ⌈count(content_items) / N⌉
+ * gives approximately {@link SAMPLE_SIZE} rows spread across the id
+ * range for large archives, and the full table for archives with
+ * ≤ {@link SAMPLE_SIZE} rows. Deterministic sampling means the same
+ * archive always produces the same verdict; a random sample would let a
+ * failing archive pass on retry if the offending rows happen to fall
+ * outside the second draw.
+ *
+ * The check throws when the sampled row count is smaller than expected
+ * (LEFT JOINs may return `null` sides but never fewer rows than the
+ * driving `content_items` set — a shortfall means SQLite silently
+ * dropped rows we intended to inspect). "Expected" is a separate
+ * `COUNT(*)` against the same `(id % stride) = 0` predicate, not a
+ * `floor(total / stride)` estimate: `content_items.id` is copied
+ * verbatim from the legacy `pages.id`, which accumulates gaps over an
+ * archive's crawl history (redirect consolidation, deleted pages, …).
+ * A density estimate over-counts whenever the id range is sparser than
+ * `[1, total]`, which produced false-positive failures on real
+ * multi-thousand-page archives.
+ * @param trx - Knex instance or transaction connected to the populated archive.
+ * @throws {MigrationVerificationError} when at least one sampled URL does not round-trip.
+ */
+export async function checkUrlRoundTrip(trx) {
+    const totalRows = await trx('content_items').count({ n: '*' });
+    const total = Number(totalRows[0].n);
+    if (total === 0) {
+        return;
+    }
+    // stride = ⌈total / SAMPLE_SIZE⌉ so `id % stride = 0` picks approximately
+    // SAMPLE_SIZE rows evenly across the id range for large archives, and the
+    // full table for archives ≤ SAMPLE_SIZE.
+    const stride = Math.max(1, Math.ceil(total / SAMPLE_SIZE));
+    const matchingRows = await trx('content_items')
+        .whereRaw('(id % ?) = 0', [stride])
+        .count({ n: '*' });
+    const expectedSize = Math.min(SAMPLE_SIZE, Number(matchingRows[0].n));
+    const sample = await trx('content_items as ci')
+        .leftJoin('pages as p', 'p.id', 'ci.id')
+        .leftJoin('url_refs as ur', 'ur.id', 'ci.url_id')
+        .select('ci.id as id', 'p.url as sourceUrl', 'ur.url as roundTripUrl')
+        .whereRaw('(ci.id % ?) = 0', [stride])
+        .orderBy('ci.id')
+        .limit(SAMPLE_SIZE);
+    if (sample.length < expectedSize) {
+        // LEFT JOINs preserve driving-table row counts (`p.id` / `url_refs.id`
+        // are PKs → 1:1 or 0:1 match). A shortfall means SQLite silently
+        // dropped rows the WHERE clause matched — a real integrity signal.
+        throw new MigrationVerificationError({
+            check: '#8 URL round-trip',
+            context: {
+                content_items_total: total,
+                stride,
+                expected_sample_size: expectedSize,
+                actual_sample_size: sample.length,
+                reason: 'sample query returned fewer rows than expected',
+            },
+        });
+    }
+    for (const row of sample) {
+        if (row.sourceUrl === null || row.roundTripUrl === null) {
+            throw new MigrationVerificationError({
+                check: '#8 URL round-trip',
+                context: {
+                    page_id: row.id,
+                    source_url: row.sourceUrl,
+                    round_trip_url: row.roundTripUrl,
+                    sample_size: sample.length,
+                    reason: 'orphan row — pages or url_refs join returned null (FK gap)',
+                },
+            });
+        }
+        if (row.sourceUrl !== row.roundTripUrl) {
+            throw new MigrationVerificationError({
+                check: '#8 URL round-trip',
+                context: {
+                    page_id: row.id,
+                    source_url: row.sourceUrl,
+                    round_trip_url: row.roundTripUrl,
+                    sample_size: sample.length,
+                },
+            });
+        }
+    }
+}

package/lib/archive/verify-migration/types.d.ts

@@ -0,0 +1,70 @@
+/**
+ * Shared type definitions for 0.13 migration verification.
+ *
+ * These are the acceptance-invariant checks that `scripts/migrate-to-0.13.mjs`
+ * runs at the end of the migration to confirm that the 0.13 entity /
+ * edge populates produced a consistent archive. Each check is one file
+ * under `verify-migration/` and throws {@link MigrationVerificationError} on mismatch;
+ * the orchestrator {@link ./verify-migration.ts} chains them in the
+ * order defined by issue #194.
+ * @module
+ */
+/**
+ * Row-count snapshot collected by every successful verify run and returned
+ * by {@link import('./verify-migration.js').verifyMigration}.
+ * The migration script logs these on the successful path so operators can
+ * eyeball migration outcomes from stdout without re-opening the archive.
+ */
+export interface MigrationVerificationSummary {
+    /** `count(*) FROM content_items` — matches `count(pages)`. */
+    readonly contentItems: number;
+    /** `count(*) FROM page_meta` — matches `count(pages WHERE scraped=1)`. */
+    readonly pageMeta: number;
+    /** `count(*) FROM anchor_edges` — always ≤ `count(anchors)`. */
+    readonly anchorEdges: number;
+    /** `SUM(count) FROM anchor_edges` — matches `count(anchors)`. */
+    readonly anchorEdgesSum: number;
+    /** `count(*) FROM image_items` — matches `count(images)`. */
+    readonly imageItems: number;
+    /** `count(*) FROM resource_items` — matches `count(resources)`. */
+    readonly resourceItems: number;
+}
+/**
+ * Structured details of a single migration invariant that failed. Attached to
+ * {@link MigrationVerificationError} so operators can see which check tripped
+ * without parsing the error message.
+ */
+export interface MigrationVerificationErrorDetails {
+    /**
+     * Human-readable check identifier matching issue #194's numbered list
+     * (e.g. `'#3'` for the anchor-edge dedup range check). Included in the
+     * error message so migration logs point at the exact clause.
+     */
+    readonly check: string;
+    /**
+     * Optional structured payload — typically `expected` / `actual` counts or
+     * the offending page id from the URL round-trip sample. Rendered into the
+     * error message so operators can diagnose without re-running the check.
+     */
+    readonly context?: Readonly<Record<string, string | number | null>>;
+}
+/**
+ * Error thrown by any 0.13 check when an invariant does not hold. The
+ * migration script catches this at the top level and aborts with a non-zero
+ * exit code; the enclosing transaction rolls back so the archive returns to
+ * its pre-populate state (ref tables from ref populate stay populated but are
+ * additive).
+ *
+ * Prefer a single error class with a structured {@link details} payload over
+ * one subclass per check: eight subclasses would create eight nearly
+ * identical constructors, and callers (script + integration test) only need
+ * to distinguish "verification failed" from other error kinds.
+ */
+export declare class MigrationVerificationError extends Error {
+    readonly details: MigrationVerificationErrorDetails;
+    /**
+     * @param details - Structured description of which check failed and why.
+     *   Rendered into the error message on construction.
+     */
+    constructor(details: MigrationVerificationErrorDetails);
+}

package/lib/archive/verify-migration/types.js

@@ -0,0 +1,63 @@
+/**
+ * Shared type definitions for 0.13 migration verification.
+ *
+ * These are the acceptance-invariant checks that `scripts/migrate-to-0.13.mjs`
+ * runs at the end of the migration to confirm that the 0.13 entity /
+ * edge populates produced a consistent archive. Each check is one file
+ * under `verify-migration/` and throws {@link MigrationVerificationError} on mismatch;
+ * the orchestrator {@link ./verify-migration.ts} chains them in the
+ * order defined by issue #194.
+ * @module
+ */
+/**
+ * Error thrown by any 0.13 check when an invariant does not hold. The
+ * migration script catches this at the top level and aborts with a non-zero
+ * exit code; the enclosing transaction rolls back so the archive returns to
+ * its pre-populate state (ref tables from ref populate stay populated but are
+ * additive).
+ *
+ * Prefer a single error class with a structured {@link details} payload over
+ * one subclass per check: eight subclasses would create eight nearly
+ * identical constructors, and callers (script + integration test) only need
+ * to distinguish "verification failed" from other error kinds.
+ */
+export class MigrationVerificationError extends Error {
+    details;
+    /**
+     * @param details - Structured description of which check failed and why.
+     *   Rendered into the error message on construction.
+     */
+    constructor(details) {
+        const contextText = details.context === undefined
+            ? ''
+            : ' — ' +
+                Object.entries(details.context)
+                    .map(([key, value]) => 
+                // Render null as the parenthesised literal `(null)` so
+                // operators reading the log can distinguish "the sample
+                // query returned no rows" from a caller that passed the
+                // four-character string `"null"`. Strings get quoted for
+                // the same reason.
+                `${key}=${formatContextValue(value)}`)
+                    .join(', ');
+        super(`migration verification failed ${details.check}${contextText}`);
+        this.details = details;
+        this.name = 'MigrationVerificationError';
+    }
+}
+/**
+ * Renders one context value for {@link MigrationVerificationError}'s message.
+ * `null` becomes the literal `(null)`; strings are wrapped in double quotes;
+ * numbers pass through as their decimal representation. The intent is to
+ * make missing-vs-present distinguishable in operator logs.
+ * @param value - Value from `MigrationVerificationErrorDetails.context`.
+ */
+function formatContextValue(value) {
+    if (value === null) {
+        return '(null)';
+    }
+    if (typeof value === 'string') {
+        return JSON.stringify(value);
+    }
+    return String(value);
+}

package/lib/archive/verify-migration/verify-migration.d.ts

@@ -0,0 +1,41 @@
+import type { MigrationVerificationSummary } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Runs every 0.13 migration acceptance invariant against a populated archive and
+ * returns the verified row-count summary on success. The migration script
+ * (`scripts/migrate-to-0.13.mjs`) calls this **inside** the same
+ * `knex.transaction()` block that ran `populateEntityTables` so a thrown
+ * {@link MigrationVerificationError} rolls back the entire entity populate — ref
+ * tables from ref populate stay committed because they live in a separate, additive
+ * transaction and never lose facts on re-run.
+ *
+ * The returned {@link MigrationVerificationSummary} is echoed to stdout by the
+ * migration script so operators can eyeball migration outcomes (`did we lose
+ * rows silently within the invariant window?`) from batch pipeline logs
+ * without re-opening the archive. Callers must run this inside a `db.transaction()`
+ * for the counts and the JOIN-based checks (#7 / #8) to see a consistent
+ * snapshot of the archive.
+ *
+ * Checks run sequentially in issue #194's numbered order so error messages
+ * pinpoint the earliest broken invariant. Sequential (rather than parallel)
+ * execution keeps the failure diagnostics focused: fanning eight COUNTs out
+ * at once would return the first-completed rejection rather than the
+ * lowest-numbered one, which surprises operators reading migration logs.
+ * The individual COUNT queries are fast enough at any real archive scale
+ * for sequential wall-clock cost to be irrelevant.
+ *
+ * Non-{@link MigrationVerificationError} exceptions from a check (e.g. a
+ * transient libsql error or a schema-drift SqliteError) are wrapped in a
+ * {@link MigrationVerificationError} so the migration script's `catch` sees a
+ * uniform failure surface — otherwise operators grepping stderr for
+ * `migration verification failed` would miss driver-side errors.
+ * @param trx - Knex instance or transaction connected to the archive
+ *   **after** every 0.13 migration populate has run. Must be a transaction
+ *   in production so ref counts, JOIN samples, and error rollback see the
+ *   same snapshot; non-transactional callers (unit tests) work but do not
+ *   get snapshot isolation.
+ * @returns Row-count summary of the verified archive.
+ * @throws {MigrationVerificationError} on the first invariant that does not
+ *   hold; subsequent checks are skipped.
+ */
+export declare function verifyMigration(trx: Knex): Promise<MigrationVerificationSummary>;

package/lib/archive/verify-migration/verify-migration.js

@@ -0,0 +1,120 @@
+import { checkAnchorEdgesCount } from './check-anchor-edges-count.js';
+import { checkAnchorEdgesSum } from './check-anchor-edges-sum.js';
+import { checkContentItemsCount } from './check-content-items-count.js';
+import { checkContentTypePreservation } from './check-content-type-preservation.js';
+import { checkImageItemsCount } from './check-image-items-count.js';
+import { checkPageMetaCount } from './check-page-meta-count.js';
+import { checkReaderParity } from './check-reader-parity.js';
+import { checkResourceItemsCount } from './check-resource-items-count.js';
+import { checkUrlRoundTrip } from './check-url-round-trip.js';
+import { MigrationVerificationError } from './types.js';
+/**
+ * Runs every 0.13 migration acceptance invariant against a populated archive and
+ * returns the verified row-count summary on success. The migration script
+ * (`scripts/migrate-to-0.13.mjs`) calls this **inside** the same
+ * `knex.transaction()` block that ran `populateEntityTables` so a thrown
+ * {@link MigrationVerificationError} rolls back the entire entity populate — ref
+ * tables from ref populate stay committed because they live in a separate, additive
+ * transaction and never lose facts on re-run.
+ *
+ * The returned {@link MigrationVerificationSummary} is echoed to stdout by the
+ * migration script so operators can eyeball migration outcomes (`did we lose
+ * rows silently within the invariant window?`) from batch pipeline logs
+ * without re-opening the archive. Callers must run this inside a `db.transaction()`
+ * for the counts and the JOIN-based checks (#7 / #8) to see a consistent
+ * snapshot of the archive.
+ *
+ * Checks run sequentially in issue #194's numbered order so error messages
+ * pinpoint the earliest broken invariant. Sequential (rather than parallel)
+ * execution keeps the failure diagnostics focused: fanning eight COUNTs out
+ * at once would return the first-completed rejection rather than the
+ * lowest-numbered one, which surprises operators reading migration logs.
+ * The individual COUNT queries are fast enough at any real archive scale
+ * for sequential wall-clock cost to be irrelevant.
+ *
+ * Non-{@link MigrationVerificationError} exceptions from a check (e.g. a
+ * transient libsql error or a schema-drift SqliteError) are wrapped in a
+ * {@link MigrationVerificationError} so the migration script's `catch` sees a
+ * uniform failure surface — otherwise operators grepping stderr for
+ * `migration verification failed` would miss driver-side errors.
+ * @param trx - Knex instance or transaction connected to the archive
+ *   **after** every 0.13 migration populate has run. Must be a transaction
+ *   in production so ref counts, JOIN samples, and error rollback see the
+ *   same snapshot; non-transactional callers (unit tests) work but do not
+ *   get snapshot isolation.
+ * @returns Row-count summary of the verified archive.
+ * @throws {MigrationVerificationError} on the first invariant that does not
+ *   hold; subsequent checks are skipped.
+ */
+export async function verifyMigration(trx) {
+    try {
+        await checkContentItemsCount(trx);
+        await checkPageMetaCount(trx);
+        await checkAnchorEdgesCount(trx);
+        await checkAnchorEdgesSum(trx);
+        await checkImageItemsCount(trx);
+        await checkResourceItemsCount(trx);
+        await checkContentTypePreservation(trx);
+        await checkUrlRoundTrip(trx);
+        // Reader-level parity between the legacy tables and the
+        // new entity tables (see `checkReaderParity` for the eight totals
+        // this catches that the row-count checks above miss).
+        await checkReaderParity(trx);
+    }
+    catch (error) {
+        if (error instanceof MigrationVerificationError) {
+            throw error;
+        }
+        throw new MigrationVerificationError({
+            check: 'runtime',
+            context: {
+                underlying_error: error instanceof Error ? error.message : String(error),
+            },
+        });
+    }
+    return collectSummary(trx);
+}
+/**
+ * Runs the six row counts that populate {@link MigrationVerificationSummary}.
+ * Separate from the invariant checks so those stay side-effect-free asserts,
+ * and this runs once at the end when all invariants have already held.
+ * @param trx - Same Knex handle passed to {@link verifyMigration}.
+ */
+async function collectSummary(trx) {
+    const [contentItems, pageMeta, anchorEdges, anchorEdgesSum, imageItems, resourceItems] = await Promise.all([
+        countOf(trx, 'content_items'),
+        countOf(trx, 'page_meta'),
+        countOf(trx, 'anchor_edges'),
+        sumOf(trx, 'anchor_edges', 'count'),
+        countOf(trx, 'image_items'),
+        countOf(trx, 'resource_items'),
+    ]);
+    return {
+        contentItems,
+        pageMeta,
+        anchorEdges,
+        anchorEdgesSum,
+        imageItems,
+        resourceItems,
+    };
+}
+/**
+ * `SELECT count(*) FROM <table>` returning a plain number.
+ * @param trx - Knex handle.
+ * @param table - Table to count.
+ */
+async function countOf(trx, table) {
+    const rows = await trx(table).count({ n: '*' });
+    return Number(rows[0].n);
+}
+/**
+ * `SELECT SUM(<column>) FROM <table>` returning a plain number (`null` sum
+ * on an empty table collapses to zero).
+ * @param trx - Knex handle.
+ * @param table - Table to sum over.
+ * @param column - Column to sum.
+ */
+async function sumOf(trx, table, column) {
+    const rows = await trx(table).sum({ n: column });
+    return Number(rows[0].n ?? 0);
+}

package/lib/classify-error-kind.d.ts

@@ -0,0 +1,19 @@
+import type { ErrorKind } from './types.js';
+/**
+ * Classify a raw crawler/scraper error message into a coarse {@link ErrorKind}.
+ *
+ * Pure and deterministic: the same message always yields the same kind, which
+ * is why the kind is derived on read rather than persisted — it can be applied
+ * uniformly to freshly captured `crawl_errors`, legacy `error.log` lines, and
+ * `page_errors` alike.
+ * @param message - The raw error message (a single line is sufficient; the
+ *   cause token such as `ENOTFOUND` or `Navigation timeout` lives there).
+ * @returns The matched kind, or `unknown` when no matcher applies.
+ * @example
+ * ```ts
+ * classifyErrorKind('getaddrinfo ENOTFOUND www.example.com'); // 'dns'
+ * classifyErrorKind('gave up after 3 retries — Race 180,000ms'); // 'timeout'
+ * classifyErrorKind('Protocol error (Page.reload): Target closed'); // 'protocol'
+ * ```
+ */
+export declare function classifyErrorKind(message: string): ErrorKind;

package/lib/classify-error-kind.js

@@ -0,0 +1,122 @@
+/**
+ * Ordered message matchers. The first pattern that matches wins, so more
+ * specific transport causes (DNS, TLS, connection-*) are tested before the
+ * broader `protocol` / `timeout` buckets — e.g. `ETIMEDOUT` must classify as
+ * `connection-timeout`, not the page-level `timeout`, and a puppeteer
+ * `Protocol error` must not be swallowed by the `timeout` matcher.
+ */
+const MATCHERS = [
+    // `dns-transient` must be evaluated before `dns`: an `EAI_AGAIN` line also
+    // carries the `getaddrinfo` token, so the more specific transient pattern
+    // has to win. Splitting it out from `dns` keeps the DNS-burned host cache
+    // (which marks on `kind === 'dns'`) from punishing a host whose only sin
+    // was a local resolver hiccup.
+    { kind: 'dns-transient', pattern: /EAI_AGAIN|\bEREFUSED\b/i },
+    {
+        kind: 'dns',
+        pattern: /ENOTFOUND|getaddrinfo|ERR_NAME_NOT_RESOLVED|ERR_NAME_RESOLUTION_FAILED/i,
+    },
+    {
+        kind: 'tls',
+        // `Hostname/IP does not match certificate's altnames` is Node's
+        // node:tls hostname mismatch error and is emphatically a TLS issue;
+        // adding it here (alongside the OpenSSL / Chromium tokens) keeps
+        // hosts that serve the wrong-name cert (common with misconfigured
+        // edge / load-balancer setups) out of `unknown`. `altnames` is
+        // anchored to the preceding `certificate` token so a request whose
+        // error message merely mentions a path containing the substring
+        // `altnames` (e.g. `https://api.example.com/altnames/lookup` in a
+        // 5xx body) does NOT get misclassified into `tls` (which is a
+        // `PERMANENT_ERROR_KINDS` member — a false-positive would
+        // permanently exclude that page from `--retry-failed`).
+        pattern: /ERR_CERT|ERR_SSL|\bCERT_|SSL routines|ERR_BAD_SSL|UNABLE_TO_VERIFY|unable to verify|self.signed certificate|certificate has expired\s*$|\bERR_TLS|Hostname\/IP does not match certificate|certificate'?s? altnames/i,
+    },
+    { kind: 'connection-refused', pattern: /ECONNREFUSED|ERR_CONNECTION_REFUSED/i },
+    {
+        kind: 'connection-reset',
+        pattern: /ECONNRESET|socket hang up|ERR_CONNECTION_RESET|ERR_CONNECTION_CLOSED|ERR_EMPTY_RESPONSE/i,
+    },
+    {
+        kind: 'connection-timeout',
+        pattern: /ETIMEDOUT|ERR_CONNECTION_TIMED_OUT|ERR_TIMED_OUT/i,
+    },
+    // `local-network` is evaluated AFTER the connection-* matchers so a
+    // concrete cause (refused / reset / timeout) wins when both apply. Only
+    // "local network is unreachable / changed" symptoms — and the OS-level
+    // errors that surface them — land here. Short tokens (`EPIPE`, `EREFUSED`)
+    // are word-bounded so unrelated identifiers don't false-positive.
+    {
+        kind: 'local-network',
+        pattern: /ERR_INTERNET_DISCONNECTED|ERR_NETWORK_CHANGED|ERR_NETWORK_IO_SUSPENDED|ERR_ADDRESS_UNREACHABLE|ERR_NETWORK_UNREACHABLE|ENETUNREACH|EHOSTUNREACH|EADDRNOTAVAIL|ENOTCONN|\bEPIPE\b/i,
+    },
+    {
+        kind: 'parse-error',
+        pattern: /Parse Error|Expected HTTP\/|Unexpected end of stream/i,
+    },
+    // `client-blocked` covers Chromium's ERR_BLOCKED_* family — the browser
+    // actively decided to reject the request (ad/tracker heuristics, CSP,
+    // CORB / ORB, administrator block list, fingerprinting protection,
+    // cleartext policy, …). Per the upstream `net/base/net_error_list.h`,
+    // `ERR_BLOCKED_BY_CLIENT` is documented as "The client chose to block
+    // the request." — i.e. the server was never the deciding party. Listed
+    // before `protocol` so puppeteer's generic "Protocol error" wrapper
+    // (which sometimes embeds the underlying net error code) is correctly
+    // attributed to the blocked layer rather than the protocol layer.
+    {
+        kind: 'client-blocked',
+        pattern: /ERR_BLOCKED_BY_CLIENT|ERR_BLOCKED_BY_ADMINISTRATOR|ERR_BLOCKED_IN_INCOGNITO_BY_ADMINISTRATOR|ERR_BLOCKED_BY_RESPONSE|ERR_BLOCKED_BY_CSP|ERR_BLOCKED_BY_ORB|ERR_BLOCKED_BY_FINGERPRINTING_PROTECTION|ERR_CLEARTEXT_NOT_PERMITTED|ERR_NETWORK_ACCESS_REVOKED/i,
+    },
+    {
+        kind: 'protocol',
+        // `detached frame` is anchored to puppeteer's exact prefix
+        // `Attempted to use detached Frame` (its current Frame.ts
+        // emitter; the `i` flag below catches the lowercase variant
+        // automatically), not the bare two-token substring. The bare
+        // form would match unrelated diagnostics like a console message
+        // "detached frame ref leaked" echoed through a logger. The older
+        // Page-domain `frame (?:was |got )?detached` form is kept as a
+        // separate alternative because Chromium still surfaces that
+        // phrasing in some legacy code paths. Without one of these, the
+        // "Attempted to use detached Frame ..." messages observed on a
+        // real archive would slip into `unknown`.
+        pattern: /Protocol error|Target closed|Session closed|Execution context was destroyed|frame (?:was |got )?detached|Attempted to use detached frame|Navigating frame was detached|Cannot find context|Node with given id|Page\.\w+ returned/i,
+    },
+    {
+        kind: 'timeout',
+        // `Timeout: https?:` matches the NetTimeoutError "Timeout: <url>"
+        // form. Looking for the URL-shaped tail (rather than anchoring at
+        // line start) is what lets us catch the beholder-wrapped variant
+        // `[Retried N times] Timeout: https://...` that gets stored in
+        // `crawl_errors` / `error.log` after retry exhaustion — the bare
+        // `^Timeout:` form would only fire on the immediate failure and
+        // miss every retry-exhausted record (the ones that actually land
+        // in the archive). Required for slow-server timeouts that
+        // previously fell into `unknown`.
+        pattern: /Race \d|Navigation timeout|timeout of \d+\s*ms exceeded|TimeoutError|Timed out|Timeout: https?:/i,
+    },
+];
+/**
+ * Classify a raw crawler/scraper error message into a coarse {@link ErrorKind}.
+ *
+ * Pure and deterministic: the same message always yields the same kind, which
+ * is why the kind is derived on read rather than persisted — it can be applied
+ * uniformly to freshly captured `crawl_errors`, legacy `error.log` lines, and
+ * `page_errors` alike.
+ * @param message - The raw error message (a single line is sufficient; the
+ *   cause token such as `ENOTFOUND` or `Navigation timeout` lives there).
+ * @returns The matched kind, or `unknown` when no matcher applies.
+ * @example
+ * ```ts
+ * classifyErrorKind('getaddrinfo ENOTFOUND www.example.com'); // 'dns'
+ * classifyErrorKind('gave up after 3 retries — Race 180,000ms'); // 'timeout'
+ * classifyErrorKind('Protocol error (Page.reload): Target closed'); // 'protocol'
+ * ```
+ */
+export function classifyErrorKind(message) {
+    for (const { kind, pattern } of MATCHERS) {
+        if (pattern.test(message)) {
+            return kind;
+        }
+    }
+    return 'unknown';
+}