@nitpicker/archive 0.0.0 → 0.25.1

package/lib/decode-stored-blob-bytes.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Decodes a stored HTML body BLOB to its UTF-8 bytes according to its codec
+ * marker, without building a JavaScript string.
+ *
+ * For consumers that hand the HTML straight to byte-oriented code (the
+ * `@nitpicker/core` native addon): going through `decodeStoredBlob`'s string
+ * would decode to UTF-16 and immediately re-encode to the same bytes, two
+ * full-size copies per page. Stored blobs are always `Buffer.from(html)`
+ * output, so the bytes are valid UTF-8 and identical to that round-trip.
+ *
+ * A zstd blob may decompress to at most `MAX_DECODED_SNAPSHOT_BYTES`, the
+ * same limit the native HTML scans apply, so a document that one path can
+ * decompress is one every path can. The error names no row; a reader that
+ * knows the page uses `decodePageHtmlBlobBytes`.
+ * @param body - Raw bytes as stored in `page_html_blobs.body`.
+ * @param codec - The `codec` column value (e.g. `'zstd'`, `'none'`).
+ * @returns The HTML as UTF-8 bytes.
+ * @throws {Error} If the codec is not recognised, or a zstd blob decompresses past the limit.
+ * @example
+ * ```ts
+ * const bytes = decodeStoredBlobBytes(row.body, row.codec);
+ * computeBodyHash(bytes);
+ * ```
+ */
+export declare function decodeStoredBlobBytes(body: Uint8Array, codec: string): Buffer;

package/lib/decode-stored-blob-bytes.js

@@ -0,0 +1,47 @@
+import { zstdDecompressSync } from 'node:zlib';
+import { MAX_DECODED_SNAPSHOT_BYTES } from '@nitpicker/core/max-decoded-snapshot-bytes';
+/**
+ * Decodes a stored HTML body BLOB to its UTF-8 bytes according to its codec
+ * marker, without building a JavaScript string.
+ *
+ * For consumers that hand the HTML straight to byte-oriented code (the
+ * `@nitpicker/core` native addon): going through `decodeStoredBlob`'s string
+ * would decode to UTF-16 and immediately re-encode to the same bytes, two
+ * full-size copies per page. Stored blobs are always `Buffer.from(html)`
+ * output, so the bytes are valid UTF-8 and identical to that round-trip.
+ *
+ * A zstd blob may decompress to at most `MAX_DECODED_SNAPSHOT_BYTES`, the
+ * same limit the native HTML scans apply, so a document that one path can
+ * decompress is one every path can. The error names no row; a reader that
+ * knows the page uses `decodePageHtmlBlobBytes`.
+ * @param body - Raw bytes as stored in `page_html_blobs.body`.
+ * @param codec - The `codec` column value (e.g. `'zstd'`, `'none'`).
+ * @returns The HTML as UTF-8 bytes.
+ * @throws {Error} If the codec is not recognised, or a zstd blob decompresses past the limit.
+ * @example
+ * ```ts
+ * const bytes = decodeStoredBlobBytes(row.body, row.codec);
+ * computeBodyHash(bytes);
+ * ```
+ */
+export function decodeStoredBlobBytes(body, codec) {
+    // `Buffer.from(buffer)` accepts Uint8Array, Buffer, and array-like
+    // shapes uniformly; libsql may hand back any of these for a BLOB
+    // column depending on the row encoding.
+    const buffer = Buffer.from(body);
+    if (codec === 'zstd') {
+        try {
+            return zstdDecompressSync(buffer, { maxOutputLength: MAX_DECODED_SNAPSHOT_BYTES });
+        }
+        catch (error) {
+            if (error.code === 'ERR_BUFFER_TOO_LARGE') {
+                throw new Error(`The page_html_blobs blob decompresses to more than ${MAX_DECODED_SNAPSHOT_BYTES} bytes, the limit on one snapshot`, { cause: error });
+            }
+            throw error;
+        }
+    }
+    if (codec === 'none') {
+        return buffer;
+    }
+    throw new Error(`Unknown page_html_blobs.codec: ${codec}`);
+}

package/lib/derive-lineage-from-parent.d.ts

@@ -0,0 +1,37 @@
+import type { PageSource } from './types.js';
+/**
+ * Decide which {@link PageSource} label a child row reached through the
+ * crawl graph (anchor placeholder, sub-resource, redirect chain
+ * intermediate) should inherit from its parent.
+ *
+ * Two simple rules, expressed once so anchor / redirect / sub-resource
+ * call sites stay in lockstep:
+ *
+ * 1. If the parent is in the inventory chain
+ *    ({@link isInventorySource}) → propagate
+ *    `'inventory-discovered'`. The child is itself a transitively
+ *    reached node in the inventory chain; it is NOT a new seed (the
+ *    seed label is reserved for URLs the operator listed in
+ *    `--inventory ./list.txt`).
+ *
+ * 2. Otherwise → return `fallback`. The two production fallbacks differ
+ *    by call site:
+ *
+ *    - Anchor lineage passes `'crawled'` explicitly so the crawled-wins
+ *      downgrade inside `resolveContentItemId` fires when the anchor reaches an
+ *      existing `'inventory-*'` row.
+ *    - Sub-resource emit passes `undefined` so the DB DEFAULT
+ *      `'crawled'` lands on the freshly INSERTed `resources` row (the
+ *      `setResources` path is INSERT-only with `onConflict.ignore()`, so
+ *      no downgrade is needed).
+ *    - Redirect chain intermediate uses `'crawled'` (same reason as
+ *      anchor): an existing inventory-* intermediate reached by a
+ *      crawled redirect chain must be downgraded.
+ *
+ * Pure function — keeps the lineage decision testable in isolation from
+ * the database transaction / event-emitter wiring that consumes it.
+ * @param parentSource - The parent page's stored `source` column (or `undefined` when no parent row exists).
+ * @param fallback - The label to return when the parent is NOT in the inventory chain. Pass `'crawled'` to enable the crawled-wins downgrade, or `undefined` to let the DB DEFAULT apply.
+ * @returns The lineage label to attach to the child row.
+ */
+export declare function deriveLineageFromParent(parentSource: PageSource | undefined, fallback: PageSource | undefined): PageSource | undefined;

package/lib/derive-lineage-from-parent.js

@@ -0,0 +1,42 @@
+import { isInventorySource } from './is-inventory-source.js';
+/**
+ * Decide which {@link PageSource} label a child row reached through the
+ * crawl graph (anchor placeholder, sub-resource, redirect chain
+ * intermediate) should inherit from its parent.
+ *
+ * Two simple rules, expressed once so anchor / redirect / sub-resource
+ * call sites stay in lockstep:
+ *
+ * 1. If the parent is in the inventory chain
+ *    ({@link isInventorySource}) → propagate
+ *    `'inventory-discovered'`. The child is itself a transitively
+ *    reached node in the inventory chain; it is NOT a new seed (the
+ *    seed label is reserved for URLs the operator listed in
+ *    `--inventory ./list.txt`).
+ *
+ * 2. Otherwise → return `fallback`. The two production fallbacks differ
+ *    by call site:
+ *
+ *    - Anchor lineage passes `'crawled'` explicitly so the crawled-wins
+ *      downgrade inside `resolveContentItemId` fires when the anchor reaches an
+ *      existing `'inventory-*'` row.
+ *    - Sub-resource emit passes `undefined` so the DB DEFAULT
+ *      `'crawled'` lands on the freshly INSERTed `resources` row (the
+ *      `setResources` path is INSERT-only with `onConflict.ignore()`, so
+ *      no downgrade is needed).
+ *    - Redirect chain intermediate uses `'crawled'` (same reason as
+ *      anchor): an existing inventory-* intermediate reached by a
+ *      crawled redirect chain must be downgraded.
+ *
+ * Pure function — keeps the lineage decision testable in isolation from
+ * the database transaction / event-emitter wiring that consumes it.
+ * @param parentSource - The parent page's stored `source` column (or `undefined` when no parent row exists).
+ * @param fallback - The label to return when the parent is NOT in the inventory chain. Pass `'crawled'` to enable the crawled-wins downgrade, or `undefined` to let the DB DEFAULT apply.
+ * @returns The lineage label to attach to the child row.
+ */
+export function deriveLineageFromParent(parentSource, fallback) {
+    if (isInventorySource(parentSource)) {
+        return 'inventory-discovered';
+    }
+    return fallback;
+}

package/lib/drop-legacy-tables.d.ts

@@ -0,0 +1,45 @@
+import type { Knex } from 'knex';
+/**
+ * Drops the five legacy write-model tables from a migrated archive. Reader
+ * and writer both operate exclusively on the entity / ref tables, so once
+ * the 0.13 populate has copied everything across (and
+ * `retargetLegacyFkTables` has repointed the adjunct FKs at
+ * `content_items`), the legacy tables are pure dead weight — dropping them
+ * reclaims the duplicated storage and removes the last consumers of the
+ * `pages(id)` id-space.
+ *
+ * **Requires `PRAGMA foreign_keys = OFF`** (asserted at runtime, because a
+ * caller that forgets the toggle would fail nondeterministically instead
+ * of loudly). Two reasons for the requirement:
+ *
+ * 1. `pages.redirectDestId` is a self-referential FK. With enforcement ON,
+ *    `DROP TABLE` performs an implicit `DELETE FROM`, and deleting a
+ *    redirect-destination row before its source row trips the constraint —
+ *    whether the drop survives depends on row order, so archives with
+ *    redirect chains can fail nondeterministically.
+ * 2. With enforcement OFF the implicit `DELETE FROM` is skipped entirely
+ *    and SQLite reclaims each table as a whole b-tree — O(pages) row
+ *    deletions become O(1) truncations, which matters at the several
+ *    hundred-thousand-row scale real archives reach.
+ *
+ * The drops still run children-first (`resources-referrers` / `anchors` /
+ * `images` reference `pages` / `resources`) as defence in depth. Missing
+ * tables are skipped so the function is idempotent.
+ *
+ * Data-integrity of the surviving tables is asserted separately by
+ * `checkForeignKeyIntegrity` (`PRAGMA foreign_key_check` ignores the
+ * `foreign_keys` setting), which the migration script runs right after
+ * this function.
+ * @param trx - Knex transaction on the archive being migrated
+ *   (`scripts/migrate-to-0.13.mjs` is the caller).
+ * @throws {Error} when the connection still has `PRAGMA foreign_keys = ON`
+ *   (`foreign_keys` cannot change inside a transaction, so the caller must
+ *   toggle it off before opening the transaction).
+ * @example
+ * await db.raw('PRAGMA foreign_keys = OFF');
+ * await db.transaction(async (trx) => {
+ *   await dropLegacyTables(trx);
+ * });
+ * await db.raw('PRAGMA foreign_keys = ON');
+ */
+export declare function dropLegacyTables(trx: Knex): Promise<void>;

package/lib/drop-legacy-tables.js

@@ -0,0 +1,56 @@
+/**
+ * Drops the five legacy write-model tables from a migrated archive. Reader
+ * and writer both operate exclusively on the entity / ref tables, so once
+ * the 0.13 populate has copied everything across (and
+ * `retargetLegacyFkTables` has repointed the adjunct FKs at
+ * `content_items`), the legacy tables are pure dead weight — dropping them
+ * reclaims the duplicated storage and removes the last consumers of the
+ * `pages(id)` id-space.
+ *
+ * **Requires `PRAGMA foreign_keys = OFF`** (asserted at runtime, because a
+ * caller that forgets the toggle would fail nondeterministically instead
+ * of loudly). Two reasons for the requirement:
+ *
+ * 1. `pages.redirectDestId` is a self-referential FK. With enforcement ON,
+ *    `DROP TABLE` performs an implicit `DELETE FROM`, and deleting a
+ *    redirect-destination row before its source row trips the constraint —
+ *    whether the drop survives depends on row order, so archives with
+ *    redirect chains can fail nondeterministically.
+ * 2. With enforcement OFF the implicit `DELETE FROM` is skipped entirely
+ *    and SQLite reclaims each table as a whole b-tree — O(pages) row
+ *    deletions become O(1) truncations, which matters at the several
+ *    hundred-thousand-row scale real archives reach.
+ *
+ * The drops still run children-first (`resources-referrers` / `anchors` /
+ * `images` reference `pages` / `resources`) as defence in depth. Missing
+ * tables are skipped so the function is idempotent.
+ *
+ * Data-integrity of the surviving tables is asserted separately by
+ * `checkForeignKeyIntegrity` (`PRAGMA foreign_key_check` ignores the
+ * `foreign_keys` setting), which the migration script runs right after
+ * this function.
+ * @param trx - Knex transaction on the archive being migrated
+ *   (`scripts/migrate-to-0.13.mjs` is the caller).
+ * @throws {Error} when the connection still has `PRAGMA foreign_keys = ON`
+ *   (`foreign_keys` cannot change inside a transaction, so the caller must
+ *   toggle it off before opening the transaction).
+ * @example
+ * await db.raw('PRAGMA foreign_keys = OFF');
+ * await db.transaction(async (trx) => {
+ *   await dropLegacyTables(trx);
+ * });
+ * await db.raw('PRAGMA foreign_keys = ON');
+ */
+export async function dropLegacyTables(trx) {
+    const pragma = await trx.raw('PRAGMA foreign_keys');
+    if (pragma[0]?.foreign_keys === 1) {
+        throw new Error('dropLegacyTables: PRAGMA foreign_keys is ON — turn it OFF before the transaction. ' +
+            'With enforcement ON, dropping `pages` runs an implicit DELETE whose row order ' +
+            'can trip the redirectDestId self-FK nondeterministically.');
+    }
+    await trx.raw('DROP TABLE IF EXISTS "resources-referrers"');
+    await trx.raw('DROP TABLE IF EXISTS "anchors"');
+    await trx.raw('DROP TABLE IF EXISTS "images"');
+    await trx.raw('DROP TABLE IF EXISTS "resources"');
+    await trx.raw('DROP TABLE IF EXISTS "pages"');
+}

package/lib/error-kind/classify-error-kind.d.ts

@@ -0,0 +1,20 @@
+import type { ErrorKind } from './types.js';
+/**
+ * Classify a raw crawler/scraper error message into a coarse {@link ErrorKind}.
+ *
+ * Pure and deterministic: the same message always yields the same kind, which
+ * is why the kind is derived on read rather than persisted — it can be applied
+ * uniformly to freshly captured `crawl_errors`, legacy `error.log` lines, and
+ * `page_errors` alike.
+ * @param message - The raw error message (a single line is sufficient; the
+ *   cause token such as `ENOTFOUND` or `Navigation timeout` lives there).
+ * @returns The matched kind, or `unknown` when no matcher applies.
+ * @example
+ * ```ts
+ * classifyErrorKind('getaddrinfo ENOTFOUND www.example.com'); // 'dns'
+ * classifyErrorKind('gave up after 3 retries — Race 180,000ms'); // 'timeout'
+ * classifyErrorKind('Protocol error (Page.reload): Target closed'); // 'protocol'
+ * classifyErrorKind('Maximum number of redirects exceeded'); // 'redirect-loop'
+ * ```
+ */
+export declare function classifyErrorKind(message: string): ErrorKind;

package/lib/error-kind/classify-error-kind.js

@@ -0,0 +1,136 @@
+/**
+ * Ordered message matchers. The first pattern that matches wins, so more
+ * specific transport causes (DNS, TLS, connection-*) are tested before the
+ * broader `protocol` / `timeout` buckets — e.g. `ETIMEDOUT` must classify as
+ * `connection-timeout`, not the page-level `timeout`, and a puppeteer
+ * `Protocol error` must not be swallowed by the `timeout` matcher.
+ */
+const MATCHERS = [
+    // `dns-transient` must be evaluated before `dns`: an `EAI_AGAIN` line also
+    // carries the `getaddrinfo` token, so the more specific transient pattern
+    // has to win. Splitting it out from `dns` keeps the DNS-burned host cache
+    // (which marks on `kind === 'dns'`) from punishing a host whose only sin
+    // was a local resolver hiccup.
+    { kind: 'dns-transient', pattern: /EAI_AGAIN|\bEREFUSED\b/i },
+    {
+        kind: 'dns',
+        pattern: /ENOTFOUND|getaddrinfo|ERR_NAME_NOT_RESOLVED|ERR_NAME_RESOLUTION_FAILED/i,
+    },
+    {
+        kind: 'tls',
+        // `Hostname/IP does not match certificate's altnames` is Node's
+        // node:tls hostname mismatch error and is emphatically a TLS issue;
+        // adding it here (alongside the OpenSSL / Chromium tokens) keeps
+        // hosts that serve the wrong-name cert (common with misconfigured
+        // edge / load-balancer setups) out of `unknown`. `altnames` is
+        // anchored to the preceding `certificate` token so a request whose
+        // error message merely mentions a path containing the substring
+        // `altnames` (e.g. `https://api.example.com/altnames/lookup` in a
+        // 5xx body) does NOT get misclassified into `tls` (which is a
+        // `PERMANENT_ERROR_KINDS` member — a false-positive would
+        // permanently exclude that page from `--retry-failed`).
+        pattern: /ERR_CERT|ERR_SSL|\bCERT_|SSL routines|ERR_BAD_SSL|UNABLE_TO_VERIFY|unable to verify|self.signed certificate|certificate has expired\s*$|\bERR_TLS|Hostname\/IP does not match certificate|certificate'?s? altnames/i,
+    },
+    { kind: 'connection-refused', pattern: /ECONNREFUSED|ERR_CONNECTION_REFUSED/i },
+    {
+        kind: 'connection-reset',
+        pattern: /ECONNRESET|socket hang up|ERR_CONNECTION_RESET|ERR_CONNECTION_CLOSED|ERR_EMPTY_RESPONSE/i,
+    },
+    {
+        kind: 'connection-timeout',
+        pattern: /ETIMEDOUT|ERR_CONNECTION_TIMED_OUT|ERR_TIMED_OUT/i,
+    },
+    // `local-network` is evaluated AFTER the connection-* matchers so a
+    // concrete cause (refused / reset / timeout) wins when both apply. Only
+    // "local network is unreachable / changed" symptoms — and the OS-level
+    // errors that surface them — land here. Short tokens (`EPIPE`, `EREFUSED`)
+    // are word-bounded so unrelated identifiers don't false-positive.
+    {
+        kind: 'local-network',
+        pattern: /ERR_INTERNET_DISCONNECTED|ERR_NETWORK_CHANGED|ERR_NETWORK_IO_SUSPENDED|ERR_ADDRESS_UNREACHABLE|ERR_NETWORK_UNREACHABLE|ENETUNREACH|EHOSTUNREACH|EADDRNOTAVAIL|ENOTCONN|\bEPIPE\b/i,
+    },
+    {
+        kind: 'parse-error',
+        pattern: /Parse Error|Expected HTTP\/|Unexpected end of stream/i,
+    },
+    // `follow-redirects` (pinned at 1.16.0, `fetch-destination.ts`'s HEAD/GET
+    // pre-flight) throws exactly "Maximum number of redirects exceeded" —
+    // no cause token, no URL — when a chain never terminates within its
+    // `maxRedirects` budget. `ERR_TOO_MANY_REDIRECTS` is the equivalent
+    // Chromium net-error code, covering the same symptom surfaced through a
+    // puppeteer navigation instead of the Node HTTP client. Both mean the
+    // SAME thing: the site's own redirect configuration loops and will loop
+    // again on any future fetch, which is why this is deterministic (not a
+    // transient network condition) — see `PERMANENT_ERROR_KINDS`.
+    {
+        kind: 'redirect-loop',
+        pattern: /Maximum number of redirects exceeded|ERR_TOO_MANY_REDIRECTS/i,
+    },
+    // `client-blocked` covers Chromium's ERR_BLOCKED_* family — the browser
+    // actively decided to reject the request (ad/tracker heuristics, CSP,
+    // CORB / ORB, administrator block list, fingerprinting protection,
+    // cleartext policy, …). Per the upstream `net/base/net_error_list.h`,
+    // `ERR_BLOCKED_BY_CLIENT` is documented as "The client chose to block
+    // the request." — i.e. the server was never the deciding party. Listed
+    // before `protocol` so puppeteer's generic "Protocol error" wrapper
+    // (which sometimes embeds the underlying net error code) is correctly
+    // attributed to the blocked layer rather than the protocol layer.
+    {
+        kind: 'client-blocked',
+        pattern: /ERR_BLOCKED_BY_CLIENT|ERR_BLOCKED_BY_ADMINISTRATOR|ERR_BLOCKED_IN_INCOGNITO_BY_ADMINISTRATOR|ERR_BLOCKED_BY_RESPONSE|ERR_BLOCKED_BY_CSP|ERR_BLOCKED_BY_ORB|ERR_BLOCKED_BY_FINGERPRINTING_PROTECTION|ERR_CLEARTEXT_NOT_PERMITTED|ERR_NETWORK_ACCESS_REVOKED/i,
+    },
+    {
+        kind: 'protocol',
+        // `detached frame` is anchored to puppeteer's exact prefix
+        // `Attempted to use detached Frame` (its current Frame.ts
+        // emitter; the `i` flag below catches the lowercase variant
+        // automatically), not the bare two-token substring. The bare
+        // form would match unrelated diagnostics like a console message
+        // "detached frame ref leaked" echoed through a logger. The older
+        // Page-domain `frame (?:was |got )?detached` form is kept as a
+        // separate alternative because Chromium still surfaces that
+        // phrasing in some legacy code paths. Without one of these, the
+        // "Attempted to use detached Frame ..." messages observed on a
+        // real archive would slip into `unknown`.
+        pattern: /Protocol error|Target closed|Session closed|Execution context was destroyed|frame (?:was |got )?detached|Attempted to use detached frame|Navigating frame was detached|Cannot find context|Node with given id|Page\.\w+ returned/i,
+    },
+    {
+        kind: 'timeout',
+        // `Timeout: https?:` matches the NetTimeoutError "Timeout: <url>"
+        // form. Looking for the URL-shaped tail (rather than anchoring at
+        // line start) is what lets us catch the beholder-wrapped variant
+        // `[Retried N times] Timeout: https://...` that gets stored in
+        // `crawl_errors` / `error.log` after retry exhaustion — the bare
+        // `^Timeout:` form would only fire on the immediate failure and
+        // miss every retry-exhausted record (the ones that actually land
+        // in the archive). Required for slow-server timeouts that
+        // previously fell into `unknown`.
+        pattern: /Race \d|Navigation timeout|timeout of \d+\s*ms exceeded|TimeoutError|Timed out|Timeout: https?:/i,
+    },
+];
+/**
+ * Classify a raw crawler/scraper error message into a coarse {@link ErrorKind}.
+ *
+ * Pure and deterministic: the same message always yields the same kind, which
+ * is why the kind is derived on read rather than persisted — it can be applied
+ * uniformly to freshly captured `crawl_errors`, legacy `error.log` lines, and
+ * `page_errors` alike.
+ * @param message - The raw error message (a single line is sufficient; the
+ *   cause token such as `ENOTFOUND` or `Navigation timeout` lives there).
+ * @returns The matched kind, or `unknown` when no matcher applies.
+ * @example
+ * ```ts
+ * classifyErrorKind('getaddrinfo ENOTFOUND www.example.com'); // 'dns'
+ * classifyErrorKind('gave up after 3 retries — Race 180,000ms'); // 'timeout'
+ * classifyErrorKind('Protocol error (Page.reload): Target closed'); // 'protocol'
+ * classifyErrorKind('Maximum number of redirects exceeded'); // 'redirect-loop'
+ * ```
+ */
+export function classifyErrorKind(message) {
+    for (const { kind, pattern } of MATCHERS) {
+        if (pattern.test(message)) {
+            return kind;
+        }
+    }
+    return 'unknown';
+}

package/lib/error-kind/network-related-error-kinds.d.ts

@@ -0,0 +1,23 @@
+import type { ErrorKind } from './types.js';
+/**
+ * `ErrorKind`s that plausibly indicate trouble with the operator's own
+ * network path rather than a genuinely dead or misconfigured target site.
+ *
+ * Single source of truth shared by two consumers that must agree on
+ * exactly the same set:
+ *
+ * - `NetworkOutageDetector` — only these kinds contribute to the sliding
+ *   window that detects a suspect outage.
+ * - `evict-network-classified-destination-cache-entries.ts` — on recovery,
+ *   only `destinationCache` entries whose cached `Error` classifies into
+ *   one of these kinds are evicted (a cached `tls` or `client-blocked`
+ *   failure is a site-specific fact and must survive an outage recovery
+ *   unrelated to it).
+ *
+ * `dns` is included despite `permanent-error-kinds.ts` treating it as a
+ * permanent, site-specific verdict in isolation — that classification is
+ * exactly what outage detection exists to override when the surrounding
+ * evidence (many hosts, tight time window) points to the local network
+ * instead.
+ */
+export declare const NETWORK_RELATED_ERROR_KINDS: ReadonlySet<ErrorKind>;

package/lib/error-kind/network-related-error-kinds.js

@@ -0,0 +1,28 @@
+/**
+ * `ErrorKind`s that plausibly indicate trouble with the operator's own
+ * network path rather than a genuinely dead or misconfigured target site.
+ *
+ * Single source of truth shared by two consumers that must agree on
+ * exactly the same set:
+ *
+ * - `NetworkOutageDetector` — only these kinds contribute to the sliding
+ *   window that detects a suspect outage.
+ * - `evict-network-classified-destination-cache-entries.ts` — on recovery,
+ *   only `destinationCache` entries whose cached `Error` classifies into
+ *   one of these kinds are evicted (a cached `tls` or `client-blocked`
+ *   failure is a site-specific fact and must survive an outage recovery
+ *   unrelated to it).
+ *
+ * `dns` is included despite `permanent-error-kinds.ts` treating it as a
+ * permanent, site-specific verdict in isolation — that classification is
+ * exactly what outage detection exists to override when the surrounding
+ * evidence (many hosts, tight time window) points to the local network
+ * instead.
+ */
+export const NETWORK_RELATED_ERROR_KINDS = new Set([
+    'dns',
+    'dns-transient',
+    'local-network',
+    'connection-timeout',
+    'connection-reset',
+]);

package/lib/error-kind/permanent-error-kinds.d.ts

@@ -0,0 +1,48 @@
+import type { ErrorKind } from './types.js';
+/**
+ * Error kinds whose failure mode is **persistent** — re-fetching the same URL
+ * (without changing the network, the certificate, the destination host, or the
+ * browser's block-list state) will almost certainly produce the same failure.
+ *
+ * Used by `resetFailedPages` to exclude pages whose latest recorded error
+ * falls in this set, so `--retry-failed` actually converges: without the
+ * exclusion, NXDOMAIN / TLS mismatch / `ERR_BLOCKED_BY_CLIENT` /
+ * `ECONNREFUSED` / HTTP parse-error / redirect-loop pages would be reset to
+ * pending on every iteration, the crawler would re-attempt them, they would
+ * fail again the same way, and the retry-target count would stay constant
+ * forever.
+ *
+ * Why these six and not others:
+ * - **dns** — `ENOTFOUND` / `ERR_NAME_NOT_RESOLVED` are authoritative DNS
+ *   answers; the host is gone (or never existed). EAI_AGAIN is split out as
+ *   `dns-transient` precisely so it is NOT in this set.
+ * - **tls** — cert expiry / SAN mismatch / SSL protocol errors require the
+ *   server operator to fix something; retrying within the same archive run
+ *   cannot succeed.
+ * - **client-blocked** — Chromium's `ERR_BLOCKED_*` family is a deliberate
+ *   browser-side rejection; the request will be blocked identically every
+ *   time the browser sees the same URL.
+ * - **parse-error** — the server's HTTP response is malformed (`Expected
+ *   HTTP/, RTSP/ or ICE/`, `Unexpected end of stream`). Retrying the same
+ *   request hits the same parser failure.
+ * - **connection-refused** — `ECONNREFUSED` is an authoritative TCP RST from
+ *   the listener; either no process is listening on the port or its accept
+ *   queue rejected the connection. Either way the answer is final until the
+ *   server operator intervenes.
+ * - **redirect-loop** — `Maximum number of redirects exceeded` /
+ *   `ERR_TOO_MANY_REDIRECTS` means the site's own redirect chain never
+ *   terminates; the exact same chain is served on every future fetch until
+ *   the site operator fixes it.
+ *
+ * Notably absent (intentionally retryable):
+ * - `connection-reset` / `connection-timeout` — could be middlebox or
+ *   transient overload
+ * - `dns-transient` (EAI_AGAIN / EREFUSED) — local resolver hiccup
+ * - `local-network` — operator-side connectivity loss
+ * - `timeout` — slow but reachable server (HEAD-timeout escalation gives
+ *   these a real chance)
+ * - `protocol` — puppeteer lifecycle race, often recovers on retry
+ * - `unknown` — by definition we don't know it's permanent, so we keep
+ *   retrying (errs on the side of investigating)
+ */
+export declare const PERMANENT_ERROR_KINDS: ReadonlySet<ErrorKind>;

package/lib/error-kind/permanent-error-kinds.js

@@ -0,0 +1,54 @@
+/**
+ * Error kinds whose failure mode is **persistent** — re-fetching the same URL
+ * (without changing the network, the certificate, the destination host, or the
+ * browser's block-list state) will almost certainly produce the same failure.
+ *
+ * Used by `resetFailedPages` to exclude pages whose latest recorded error
+ * falls in this set, so `--retry-failed` actually converges: without the
+ * exclusion, NXDOMAIN / TLS mismatch / `ERR_BLOCKED_BY_CLIENT` /
+ * `ECONNREFUSED` / HTTP parse-error / redirect-loop pages would be reset to
+ * pending on every iteration, the crawler would re-attempt them, they would
+ * fail again the same way, and the retry-target count would stay constant
+ * forever.
+ *
+ * Why these six and not others:
+ * - **dns** — `ENOTFOUND` / `ERR_NAME_NOT_RESOLVED` are authoritative DNS
+ *   answers; the host is gone (or never existed). EAI_AGAIN is split out as
+ *   `dns-transient` precisely so it is NOT in this set.
+ * - **tls** — cert expiry / SAN mismatch / SSL protocol errors require the
+ *   server operator to fix something; retrying within the same archive run
+ *   cannot succeed.
+ * - **client-blocked** — Chromium's `ERR_BLOCKED_*` family is a deliberate
+ *   browser-side rejection; the request will be blocked identically every
+ *   time the browser sees the same URL.
+ * - **parse-error** — the server's HTTP response is malformed (`Expected
+ *   HTTP/, RTSP/ or ICE/`, `Unexpected end of stream`). Retrying the same
+ *   request hits the same parser failure.
+ * - **connection-refused** — `ECONNREFUSED` is an authoritative TCP RST from
+ *   the listener; either no process is listening on the port or its accept
+ *   queue rejected the connection. Either way the answer is final until the
+ *   server operator intervenes.
+ * - **redirect-loop** — `Maximum number of redirects exceeded` /
+ *   `ERR_TOO_MANY_REDIRECTS` means the site's own redirect chain never
+ *   terminates; the exact same chain is served on every future fetch until
+ *   the site operator fixes it.
+ *
+ * Notably absent (intentionally retryable):
+ * - `connection-reset` / `connection-timeout` — could be middlebox or
+ *   transient overload
+ * - `dns-transient` (EAI_AGAIN / EREFUSED) — local resolver hiccup
+ * - `local-network` — operator-side connectivity loss
+ * - `timeout` — slow but reachable server (HEAD-timeout escalation gives
+ *   these a real chance)
+ * - `protocol` — puppeteer lifecycle race, often recovers on retry
+ * - `unknown` — by definition we don't know it's permanent, so we keep
+ *   retrying (errs on the side of investigating)
+ */
+export const PERMANENT_ERROR_KINDS = new Set([
+    'dns',
+    'tls',
+    'client-blocked',
+    'parse-error',
+    'connection-refused',
+    'redirect-loop',
+]);

package/lib/error-kind/types.d.ts

@@ -0,0 +1,54 @@
+/**
+ * Coarse cause of a crawl/scrape failure.
+ *
+ * The crawler stores only the raw error message (in `crawl_errors`,
+ * `page_errors`, or `error.log`); the cause is derived on read by
+ * `classifyErrorKind`, so existing archives gain classification without a
+ * re-crawl.
+ *
+ * Owned by the archive package because the crawler (for DNS-burned host
+ * caching), the archive layer itself (`--retry-failed` candidate selection)
+ * and `@nitpicker/query` (for `getErrorKinds` / `getSummary`) all need to
+ * classify error messages, and both crawler and query depend on archive
+ * (crawler cannot depend on query).
+ *
+ * ### transient vs persistent
+ *
+ * | kind | transient? | DNS-burn? | notes |
+ * | --- | --- | --- | --- |
+ * | `dns` | no | yes | NXDOMAIN; the host does not resolve at all |
+ * | `dns-transient` | **yes** | no | `EAI_AGAIN`; local resolver hiccup, retry often recovers |
+ * | `tls` | no | no | certificate issue, usually persistent until cert rotates |
+ * | `connection-refused` | mostly persistent | no | server actively rejecting on this port |
+ * | `connection-reset` | yes | no | TCP reset mid-stream, often transient |
+ * | `connection-timeout` | yes | no | TCP-level timeout (`ETIMEDOUT`); slow but reachable |
+ * | `local-network` | **yes** | no | local machine's network is unreachable / changed (WiFi, sleep, ICMP-unreachable, …) |
+ * | `parse-error` | mostly persistent | no | HTTP response could not be parsed (proxy, garbage, MITM) |
+ * | `client-blocked` | persistent (per browser) | no | Chromium-side `ERR_BLOCKED_BY_*` family — the browser actively refused the request (ad/tracker heuristics, CSP, CORP, administrator block list, …) |
+ * | `redirect-loop` | no | no | the redirect chain exceeded `follow-redirects`' `maxRedirects` limit — the site's own redirect configuration never converges |
+ * | `protocol` | yes | no | puppeteer protocol layer (frame detached, target closed, …) |
+ * | `timeout` | yes | no | puppeteer navigation timeout or HEAD pre-flight race timeout (`Timeout: <url>`) |
+ * | `unknown` | unknown | no | catch-all for messages no matcher recognised |
+ *
+ * Only `dns` is mark-target for the DNS-burned host cache; everything else is
+ * either too transient to burn (network glitch / browser hiccup) or too
+ * server-specific to extrapolate to "this whole host is dead."
+ *
+ * ### Derived constants that MUST be reviewed when this union changes
+ *
+ * - `PERMANENT_ERROR_KINDS` (`permanent-error-kinds.ts`) — the set of kinds
+ *   excluded from `--retry-failed` so retry iterations actually converge.
+ *   A new kind that is deterministically permanent (server-state, cert,
+ *   browser-block, …) likely belongs here.
+ * - `PUPPETEER_FALLBACK_KINDS` (`@nitpicker/crawler`'s
+ *   `crawler/is-puppeteer-fallback-candidate.ts`) — the set of kinds where
+ *   one puppeteer attempt has a realistic chance of succeeding after
+ *   HEAD+GET pre-flight exhausted retries. A new kind modelling a
+ *   middlebox / WAF / slow-server quirk likely belongs here.
+ *
+ * Adding a kind without reviewing both sets risks a silent regression:
+ * `--retry-failed` re-trying a permanent failure forever (no PERMANENT
+ * entry), or a recoverable URL never reaching the puppeteer fallback (no
+ * PUPPETEER_FALLBACK entry).
+ */
+export type ErrorKind = 'dns' | 'dns-transient' | 'connection-refused' | 'connection-reset' | 'connection-timeout' | 'tls' | 'local-network' | 'parse-error' | 'client-blocked' | 'redirect-loop' | 'timeout' | 'protocol' | 'unknown';

package/lib/error-kind/types.js

@@ -0,0 +1 @@
+export {};

package/lib/filesystem/append-text.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Appends text data to a file at the specified path.
+ *
+ * Creates parent directories if they do not exist.
+ * A newline character is prepended to the data before appending.
+ * @param filePath - The absolute or relative path to the file to append to.
+ * @param data - The text content to append to the file.
+ */
+export declare function appendText(filePath: string, data: string): Promise<void>;