@nitpicker/crawler 0.11.0 → 0.13.0

package/lib/permanent-error-kinds.d.ts

@@ -0,0 +1,43 @@
+import type { ErrorKind } from './types.js';
+/**
+ * Error kinds whose failure mode is **persistent** — re-fetching the same URL
+ * (without changing the network, the certificate, the destination host, or the
+ * browser's block-list state) will almost certainly produce the same failure.
+ *
+ * Used by `resetFailedPages` to exclude pages whose latest recorded error
+ * falls in this set, so `--retry-failed` actually converges: without the
+ * exclusion, NXDOMAIN / TLS mismatch / `ERR_BLOCKED_BY_CLIENT` /
+ * `ECONNREFUSED` / HTTP parse-error pages would be reset to pending on every
+ * iteration, the crawler would re-attempt them, they would fail again the
+ * same way, and the retry-target count would stay constant forever.
+ *
+ * Why these five and not others:
+ * - **dns** — `ENOTFOUND` / `ERR_NAME_NOT_RESOLVED` are authoritative DNS
+ *   answers; the host is gone (or never existed). EAI_AGAIN is split out as
+ *   `dns-transient` precisely so it is NOT in this set.
+ * - **tls** — cert expiry / SAN mismatch / SSL protocol errors require the
+ *   server operator to fix something; retrying within the same archive run
+ *   cannot succeed.
+ * - **client-blocked** — Chromium's `ERR_BLOCKED_*` family is a deliberate
+ *   browser-side rejection; the request will be blocked identically every
+ *   time the browser sees the same URL.
+ * - **parse-error** — the server's HTTP response is malformed (`Expected
+ *   HTTP/, RTSP/ or ICE/`, `Unexpected end of stream`). Retrying the same
+ *   request hits the same parser failure.
+ * - **connection-refused** — `ECONNREFUSED` is an authoritative TCP RST from
+ *   the listener; either no process is listening on the port or its accept
+ *   queue rejected the connection. Either way the answer is final until the
+ *   server operator intervenes.
+ *
+ * Notably absent (intentionally retryable):
+ * - `connection-reset` / `connection-timeout` — could be middlebox or
+ *   transient overload
+ * - `dns-transient` (EAI_AGAIN / EREFUSED) — local resolver hiccup
+ * - `local-network` — operator-side connectivity loss
+ * - `timeout` — slow but reachable server (HEAD-timeout escalation gives
+ *   these a real chance)
+ * - `protocol` — puppeteer lifecycle race, often recovers on retry
+ * - `unknown` — by definition we don't know it's permanent, so we keep
+ *   retrying (errs on the side of investigating)
+ */
+export declare const PERMANENT_ERROR_KINDS: ReadonlySet<ErrorKind>;

package/lib/permanent-error-kinds.js

@@ -0,0 +1,48 @@
+/**
+ * Error kinds whose failure mode is **persistent** — re-fetching the same URL
+ * (without changing the network, the certificate, the destination host, or the
+ * browser's block-list state) will almost certainly produce the same failure.
+ *
+ * Used by `resetFailedPages` to exclude pages whose latest recorded error
+ * falls in this set, so `--retry-failed` actually converges: without the
+ * exclusion, NXDOMAIN / TLS mismatch / `ERR_BLOCKED_BY_CLIENT` /
+ * `ECONNREFUSED` / HTTP parse-error pages would be reset to pending on every
+ * iteration, the crawler would re-attempt them, they would fail again the
+ * same way, and the retry-target count would stay constant forever.
+ *
+ * Why these five and not others:
+ * - **dns** — `ENOTFOUND` / `ERR_NAME_NOT_RESOLVED` are authoritative DNS
+ *   answers; the host is gone (or never existed). EAI_AGAIN is split out as
+ *   `dns-transient` precisely so it is NOT in this set.
+ * - **tls** — cert expiry / SAN mismatch / SSL protocol errors require the
+ *   server operator to fix something; retrying within the same archive run
+ *   cannot succeed.
+ * - **client-blocked** — Chromium's `ERR_BLOCKED_*` family is a deliberate
+ *   browser-side rejection; the request will be blocked identically every
+ *   time the browser sees the same URL.
+ * - **parse-error** — the server's HTTP response is malformed (`Expected
+ *   HTTP/, RTSP/ or ICE/`, `Unexpected end of stream`). Retrying the same
+ *   request hits the same parser failure.
+ * - **connection-refused** — `ECONNREFUSED` is an authoritative TCP RST from
+ *   the listener; either no process is listening on the port or its accept
+ *   queue rejected the connection. Either way the answer is final until the
+ *   server operator intervenes.
+ *
+ * Notably absent (intentionally retryable):
+ * - `connection-reset` / `connection-timeout` — could be middlebox or
+ *   transient overload
+ * - `dns-transient` (EAI_AGAIN / EREFUSED) — local resolver hiccup
+ * - `local-network` — operator-side connectivity loss
+ * - `timeout` — slow but reachable server (HEAD-timeout escalation gives
+ *   these a real chance)
+ * - `protocol` — puppeteer lifecycle race, often recovers on retry
+ * - `unknown` — by definition we don't know it's permanent, so we keep
+ *   retrying (errs on the side of investigating)
+ */
+export const PERMANENT_ERROR_KINDS = new Set([
+    'dns',
+    'tls',
+    'client-blocked',
+    'parse-error',
+    'connection-refused',
+]);

package/lib/types.d.ts

@@ -1,4 +1,88 @@
 import type { CrawlerError, PageData } from './utils/types/types.js';
+/**
+ * Aggregate counts captured during a `--inventory` invocation, forwarded to
+ * `#writeInventoryRunRow` so the audit log row is consistent between the
+ * HTML-seed branch and the non-HTML-only branch of
+ * `CrawlerOrchestrator.inventory`.
+ *
+ * Spelled out here (not inlined at the call site) so a new field added to
+ * the audit row has a single edit point and so each field's semantics are
+ * documented per-property rather than scattered across the two emit sites.
+ */
+export interface InventoryRunAggregates {
+    /** Total non-empty lines in the input list (= `inventoryUrls.length` before any filtering). Stored verbatim as `inventory_runs.total_lines`. */
+    inventoryUrlsCount: number;
+    /** Number of novel URLs classified as HTML and queued for render. Stored as `new_pages` (excludes anchor-discovered descendants — those add later via the crawler graph and are NOT counted here). */
+    htmlSeedsCount: number;
+    /** Number of novel URLs classified as non-HTML and written directly into `resources`. Stored as `new_resources`. */
+    nonHtmlCount: number;
+    /** URLs dropped because they fell outside the archived scope. Stored as `scope_skipped`. */
+    outOfScope: number;
+    /**
+     * SHA-256 hex digest of the source `.txt`, **pre-computed by the caller**
+     * (typically the CLI's `inventoryCrawl`). Stored verbatim as
+     * `inventory_runs.source_file_sha256`.
+     *
+     * Pre-computation lifts the absolute path off the orchestrator
+     * boundary entirely — the path is privacy-sensitive (leaks
+     * user-home / OS structure when archives are shared), and the
+     * orchestrator has no business handling it after the audit-row
+     * column was dropped. Pass `null` for programmatic callers that
+     * built `inventoryUrls` in-memory; the audit row's
+     * `source_file_sha256` will be `NULL`.
+     */
+    sourceFileSha256: string | null;
+}
+/**
+ * Coarse cause of a crawl/scrape failure.
+ *
+ * The crawler stores only the raw error message (in `crawl_errors`,
+ * `page_errors`, or `error.log`); the cause is derived on read by
+ * `classifyErrorKind`, so existing archives gain classification without a
+ * re-crawl.
+ *
+ * Owned by the crawler package because both the crawler (for DNS-burned host
+ * caching) and `@nitpicker/query` (for `getErrorKinds` / `getSummary`) need to
+ * classify error messages, and crawler cannot depend on query.
+ *
+ * ### transient vs persistent
+ *
+ * | kind | transient? | DNS-burn? | notes |
+ * | --- | --- | --- | --- |
+ * | `dns` | no | yes | NXDOMAIN; the host does not resolve at all |
+ * | `dns-transient` | **yes** | no | `EAI_AGAIN`; local resolver hiccup, retry often recovers |
+ * | `tls` | no | no | certificate issue, usually persistent until cert rotates |
+ * | `connection-refused` | mostly persistent | no | server actively rejecting on this port |
+ * | `connection-reset` | yes | no | TCP reset mid-stream, often transient |
+ * | `connection-timeout` | yes | no | TCP-level timeout (`ETIMEDOUT`); slow but reachable |
+ * | `local-network` | **yes** | no | local machine's network is unreachable / changed (WiFi, sleep, ICMP-unreachable, …) |
+ * | `parse-error` | mostly persistent | no | HTTP response could not be parsed (proxy, garbage, MITM) |
+ * | `client-blocked` | persistent (per browser) | no | Chromium-side `ERR_BLOCKED_BY_*` family — the browser actively refused the request (ad/tracker heuristics, CSP, CORP, administrator block list, …) |
+ * | `protocol` | yes | no | puppeteer protocol layer (frame detached, target closed, …) |
+ * | `timeout` | yes | no | puppeteer navigation timeout or HEAD pre-flight race timeout (`Timeout: <url>`) |
+ * | `unknown` | unknown | no | catch-all for messages no matcher recognised |
+ *
+ * Only `dns` is mark-target for the DNS-burned host cache; everything else is
+ * either too transient to burn (network glitch / browser hiccup) or too
+ * server-specific to extrapolate to "this whole host is dead."
+ *
+ * ### Derived constants that MUST be reviewed when this union changes
+ *
+ * - `PERMANENT_ERROR_KINDS` (`permanent-error-kinds.ts`) — the set of kinds
+ *   excluded from `--retry-failed` so retry iterations actually converge.
+ *   A new kind that is deterministically permanent (server-state, cert,
+ *   browser-block, …) likely belongs here.
+ * - `PUPPETEER_FALLBACK_KINDS` (`crawler/is-puppeteer-fallback-candidate.ts`)
+ *   — the set of kinds where one puppeteer attempt has a realistic chance
+ *   of succeeding after HEAD+GET pre-flight exhausted retries. A new kind
+ *   modelling a middlebox / WAF / slow-server quirk likely belongs here.
+ *
+ * Adding a kind without reviewing both sets risks a silent regression:
+ * `--retry-failed` re-trying a permanent failure forever (no PERMANENT
+ * entry), or a recoverable URL never reaching the puppeteer fallback (no
+ * PUPPETEER_FALLBACK entry).
+ */
+export type ErrorKind = 'dns' | 'dns-transient' | 'connection-refused' | 'connection-reset' | 'connection-timeout' | 'tls' | 'local-network' | 'parse-error' | 'client-blocked' | 'timeout' | 'protocol' | 'unknown';
 /**
  * Event map for the `CrawlerOrchestrator` class.
  *

package/lib/utils/compute-file-sha256.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Compute the SHA-256 digest of a file's bytes, streaming so memory stays
+ * O(1) regardless of file size.
+ *
+ * Used by `CrawlerOrchestrator.inventory` to fingerprint the source
+ * URL list `.txt` and store the digest on the `inventory_runs` row. The
+ * digest is the content-identity key for "was this exact list applied
+ * before" — recorded for operator audit; nothing dedupes on it at write
+ * time.
+ *
+ * Returns `null` instead of throwing when the file cannot be read (e.g.
+ * vanished mid-run, permissions issue) so the inventory run's audit row
+ * can still be written with a NULL digest. Hashing failure is an audit
+ * loss, not a correctness failure — the actual ingestion has already
+ * succeeded by the time this is called.
+ * @param filePath - Absolute or relative path to the file to hash.
+ * @returns Lower-case hex digest (64 chars), or `null` if reading failed.
+ * @example
+ * ```ts
+ * const sha = await computeFileSha256('/tmp/list.txt');
+ * if (sha) console.log(sha); // 64-char hex string
+ * ```
+ */
+export declare function computeFileSha256(filePath: string): Promise<string | null>;

package/lib/utils/compute-file-sha256.js

@@ -0,0 +1,56 @@
+import { createHash } from 'node:crypto';
+import { createReadStream } from 'node:fs';
+import { crawlerLog } from '../debug.js';
+/**
+ * Compute the SHA-256 digest of a file's bytes, streaming so memory stays
+ * O(1) regardless of file size.
+ *
+ * Used by `CrawlerOrchestrator.inventory` to fingerprint the source
+ * URL list `.txt` and store the digest on the `inventory_runs` row. The
+ * digest is the content-identity key for "was this exact list applied
+ * before" — recorded for operator audit; nothing dedupes on it at write
+ * time.
+ *
+ * Returns `null` instead of throwing when the file cannot be read (e.g.
+ * vanished mid-run, permissions issue) so the inventory run's audit row
+ * can still be written with a NULL digest. Hashing failure is an audit
+ * loss, not a correctness failure — the actual ingestion has already
+ * succeeded by the time this is called.
+ * @param filePath - Absolute or relative path to the file to hash.
+ * @returns Lower-case hex digest (64 chars), or `null` if reading failed.
+ * @example
+ * ```ts
+ * const sha = await computeFileSha256('/tmp/list.txt');
+ * if (sha) console.log(sha); // 64-char hex string
+ * ```
+ */
+export async function computeFileSha256(filePath) {
+    return await new Promise((resolve) => {
+        const hash = createHash('sha256');
+        const stream = createReadStream(filePath);
+        let settled = false;
+        const settle = (value) => {
+            if (settled) {
+                return;
+            }
+            settled = true;
+            resolve(value);
+        };
+        stream.on('data', (chunk) => hash.update(chunk));
+        stream.on('end', () => settle(hash.digest('hex')));
+        stream.on('error', (error) => {
+            // Trace the failure so operators inspecting `--inventory`
+            // audit rows with `source_file_sha256 = NULL` can recover
+            // the underlying cause via `DEBUG=Nitpicker:Crawler:*`.
+            crawlerLog('compute-file-sha256 failed for %s: %s', filePath, error.message);
+            settle(null);
+        });
+        // `'close'` is the last-resort settler. If the stream is
+        // destroyed externally between `'data'` and `'end'` (e.g. a
+        // test or signal handler), neither `'end'` nor `'error'` may
+        // fire — leaving the promise pending forever. Falling back to
+        // `null` matches the documented contract ("hashing failure
+        // yields null").
+        stream.on('close', () => settle(null));
+    });
+}

package/lib/utils/error/emit-error-with-retry.d.ts

@@ -0,0 +1,40 @@
+import type { RetryCallOptions } from '@d-zero/shared/retry';
+/**
+ * Minimal object shape required by {@link emitErrorAndRetry}: an emitter
+ * able to publish an `'error'` event carrying an `Error` payload. Same
+ * contract as `emitError` — see {@link ./emit-error.js} for details.
+ */
+interface Emitter {
+    /**
+     * Emits the `'error'` event with the given `Error` payload.
+     * @param event - Event name; always `'error'` here.
+     * @param payload - The error instance to publish.
+     * @returns Anything; the caller ignores it.
+     */
+    emit(event: 'error', payload: Error): unknown;
+}
+/**
+ * Retries `fn` per `retryOptions`, then — if retries are exhausted with a
+ * final `Error` — logs and re-emits it as an `'error'` event on `emitter`
+ * before re-throwing. `label` is forwarded into `retryCall` so its own
+ * onWait / onGiveUp / timeout messages identify the call site (previously
+ * derived automatically by the `@retry` decorator via
+ * `ClassMethodDecoratorContext.name`).
+ *
+ * Inlined try/catch rather than composing `emitError` with a
+ * `() => retryCall(...)` closure to keep the per-invocation closure count
+ * at 1 — this method is called on the DB write hot path (`updatePage`,
+ * `insertResource`), where wrapping decorators previously ran with zero
+ * per-call allocation.
+ * @template T - The return type of `fn`.
+ * @param emitter - The event emitter that receives the `'error'` event.
+ * @param label - Human-readable label; forwarded into `retryCall`
+ *                (visible in retry timeouts / onWait / onGiveUp) and used
+ *                as the debug log prefix on final failure.
+ * @param fn - The async operation to retry.
+ * @param retryOptions - Retry configuration; `label` is overwritten
+ *                       with the `label` argument here.
+ * @returns The resolved value of `fn`.
+ */
+export declare function emitErrorAndRetry<T>(emitter: Emitter, label: string, fn: () => Promise<T>, retryOptions: RetryCallOptions): Promise<T>;
+export {};

package/lib/utils/error/emit-error-with-retry.js

@@ -0,0 +1,44 @@
+import { retryCall } from '@d-zero/shared/retry';
+import { log } from '../debug.js';
+/**
+ * Debug logger for the ErrorEmitter helper family.
+ * Namespace: `Nitpicker:Utils:ErrorEmitter`. Kept identical to the namespace
+ * used by the deleted `@ErrorEmitter()` decorator so existing `DEBUG` env
+ * filters keep working after the decorator-to-HOF migration.
+ */
+const errorLog = log.extend('ErrorEmitter');
+/**
+ * Retries `fn` per `retryOptions`, then — if retries are exhausted with a
+ * final `Error` — logs and re-emits it as an `'error'` event on `emitter`
+ * before re-throwing. `label` is forwarded into `retryCall` so its own
+ * onWait / onGiveUp / timeout messages identify the call site (previously
+ * derived automatically by the `@retry` decorator via
+ * `ClassMethodDecoratorContext.name`).
+ *
+ * Inlined try/catch rather than composing `emitError` with a
+ * `() => retryCall(...)` closure to keep the per-invocation closure count
+ * at 1 — this method is called on the DB write hot path (`updatePage`,
+ * `insertResource`), where wrapping decorators previously ran with zero
+ * per-call allocation.
+ * @template T - The return type of `fn`.
+ * @param emitter - The event emitter that receives the `'error'` event.
+ * @param label - Human-readable label; forwarded into `retryCall`
+ *                (visible in retry timeouts / onWait / onGiveUp) and used
+ *                as the debug log prefix on final failure.
+ * @param fn - The async operation to retry.
+ * @param retryOptions - Retry configuration; `label` is overwritten
+ *                       with the `label` argument here.
+ * @returns The resolved value of `fn`.
+ */
+export async function emitErrorAndRetry(emitter, label, fn, retryOptions) {
+    try {
+        return await retryCall(fn, { ...retryOptions, label });
+    }
+    catch (error) {
+        if (error instanceof Error) {
+            errorLog('%s: %O', label, error);
+            void emitter.emit('error', error);
+        }
+        throw error;
+    }
+}

package/lib/utils/error/emit-error.d.ts

@@ -0,0 +1,39 @@
+/**
+ * Minimal object shape required by {@link emitError}: an emitter able to
+ * publish an `'error'` event carrying an `Error` payload. Any class that
+ * extends `TypedAwaitEventEmitter<{ error: Error, ... }>` satisfies this,
+ * and so does Node's built-in `EventEmitter`.
+ */
+interface Emitter {
+    /**
+     * Emits the `'error'` event with the given `Error` payload.
+     * The return value is intentionally unspecified — callers ignore it.
+     * @param event - Event name; always `'error'` here.
+     * @param payload - The error instance to publish.
+     * @returns Anything; the caller ignores it.
+     */
+    emit(event: 'error', payload: Error): unknown;
+}
+/**
+ * Wraps `fn` so that any thrown `Error` is logged with `label` and
+ * re-emitted as an `'error'` event on `emitter` before being re-thrown.
+ * Non-`Error` throws pass through unchanged — no log, no emit — matching
+ * the behaviour of the deleted `@ErrorEmitter()` decorator so that
+ * downstream `'error'` listeners (e.g. the crawler orchestrator that
+ * aborts on any emitted error) are only invoked with real `Error`
+ * instances.
+ *
+ * Kept as a standalone HOF (instead of a decorator) because Vite 8 /
+ * Vitest 4.1 switched from esbuild to Rolldown + oxc, which emit
+ * TC39 Stage 3 decorator syntax as-is and let it reach Node, causing
+ * `SyntaxError: Invalid or unexpected token` at test runtime.
+ * @template T - The return type of `fn`.
+ * @param emitter - The event emitter that receives the `'error'` event.
+ * @param label - Human-readable label included in the debug log line
+ *                (typically `'ClassName.methodName'`). Also useful as the
+ *                grep target when triaging failed crawls.
+ * @param fn - The async operation to wrap.
+ * @returns The resolved value of `fn`.
+ */
+export declare function emitError<T>(emitter: Emitter, label: string, fn: () => Promise<T>): Promise<T>;
+export {};

package/lib/utils/error/emit-error.js

@@ -0,0 +1,41 @@
+import { log } from '../debug.js';
+/**
+ * Debug logger for the ErrorEmitter helper family.
+ * Namespace: `Nitpicker:Utils:ErrorEmitter`. Kept identical to the namespace
+ * used by the deleted `@ErrorEmitter()` decorator so existing `DEBUG` env
+ * filters keep working after the decorator-to-HOF migration.
+ */
+const errorLog = log.extend('ErrorEmitter');
+/**
+ * Wraps `fn` so that any thrown `Error` is logged with `label` and
+ * re-emitted as an `'error'` event on `emitter` before being re-thrown.
+ * Non-`Error` throws pass through unchanged — no log, no emit — matching
+ * the behaviour of the deleted `@ErrorEmitter()` decorator so that
+ * downstream `'error'` listeners (e.g. the crawler orchestrator that
+ * aborts on any emitted error) are only invoked with real `Error`
+ * instances.
+ *
+ * Kept as a standalone HOF (instead of a decorator) because Vite 8 /
+ * Vitest 4.1 switched from esbuild to Rolldown + oxc, which emit
+ * TC39 Stage 3 decorator syntax as-is and let it reach Node, causing
+ * `SyntaxError: Invalid or unexpected token` at test runtime.
+ * @template T - The return type of `fn`.
+ * @param emitter - The event emitter that receives the `'error'` event.
+ * @param label - Human-readable label included in the debug log line
+ *                (typically `'ClassName.methodName'`). Also useful as the
+ *                grep target when triaging failed crawls.
+ * @param fn - The async operation to wrap.
+ * @returns The resolved value of `fn`.
+ */
+export async function emitError(emitter, label, fn) {
+    try {
+        return await fn();
+    }
+    catch (error) {
+        if (error instanceof Error) {
+            errorLog('%s: %O', label, error);
+            void emitter.emit('error', error);
+        }
+        throw error;
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@nitpicker/crawler",
-	"version": "0.11.0",
+	"version": "0.13.0",
 	"description": "Web crawler engine with headless browser rendering and archive storage",
 	"author": "D-ZERO",
 	"license": "Apache-2.0",
@@ -27,26 +27,26 @@
 		"clean": "tsc --build --clean"
 	},
 	"dependencies": {
-		"@d-zero/beholder": "3.1.1",
-		"@d-zero/dealer": "1.9.0",
-		"@d-zero/fs": "0.2.2",
-		"@d-zero/shared": "0.20.1",
+		"@d-zero/beholder": "3.1.5",
+		"@d-zero/dealer": "1.9.4",
+		"@d-zero/fs": "0.2.6",
+		"@d-zero/shared": "0.22.2",
 		"ansi-colors": "4.1.3",
 		"debug": "4.4.3",
-		"follow-redirects": "1.15.11",
-		"fs-extra": "11.3.3",
-		"knex": "3.1.0",
+		"follow-redirects": "1.16.0",
+		"fs-extra": "11.3.6",
+		"knex": "3.3.0",
 		"libsql": "0.5.29",
-		"puppeteer": "24.37.5",
+		"puppeteer": "25.3.0",
 		"robots-parser": "3.0.1",
-		"tar": "7.5.9"
+		"tar": "7.5.20"
 	},
 	"devDependencies": {
-		"@types/debug": "4.1.12",
+		"@types/debug": "4.1.13",
 		"@types/follow-redirects": "1.14.4",
 		"@types/fs-extra": "11.0.4",
 		"@types/tar": "7.0.87",
 		"@types/unzipper": "0.10.11"
 	},
-	"gitHead": "b11dbe3691746b3cb980476808d1f745bc6d0e98"
+	"gitHead": "2cd462ea6b183f4462b7caba27b4db04aaba7c70"
 }

package/lib/archive/migrate-crawl-errors.d.ts

@@ -1,20 +0,0 @@
-import type { Knex } from 'knex';
-/**
- * Adds the `crawl_errors` table to archives created before structured capture
- * of the crawler-level `error` channel landed.
- *
- * `crawl_errors` records errors that previously only reached `error.log` (DNS
- * failures, connection resets, TLS problems, process-level errors), in a
- * queryable form so the `error-kinds` analysis can classify them without
- * parsing the text log. Older `.nitpicker` files predate this table; this
- * migration creates it idempotently. Existing archives stay empty until they
- * are crawled again (resume / append / retry), at which point new errors are
- * written here — analysis of pre-existing data falls back to parsing
- * `error.log`.
- *
- * Idempotent: when the table already exists, the function exits without
- * touching the schema or writing to stderr. It also skips empty archives (no
- * `pages` table), which the regular initSchema path will fully provision.
- * @param instance - The Knex query builder instance connected to the database.
- */
-export declare function migrateCrawlErrors(instance: Knex): Promise<void>;

package/lib/archive/migrate-crawl-errors.js

@@ -1,38 +0,0 @@
-/**
- * Adds the `crawl_errors` table to archives created before structured capture
- * of the crawler-level `error` channel landed.
- *
- * `crawl_errors` records errors that previously only reached `error.log` (DNS
- * failures, connection resets, TLS problems, process-level errors), in a
- * queryable form so the `error-kinds` analysis can classify them without
- * parsing the text log. Older `.nitpicker` files predate this table; this
- * migration creates it idempotently. Existing archives stay empty until they
- * are crawled again (resume / append / retry), at which point new errors are
- * written here — analysis of pre-existing data falls back to parsing
- * `error.log`.
- *
- * Idempotent: when the table already exists, the function exits without
- * touching the schema or writing to stderr. It also skips empty archives (no
- * `pages` table), which the regular initSchema path will fully provision.
- * @param instance - The Knex query builder instance connected to the database.
- */
-export async function migrateCrawlErrors(instance) {
-    const hasTable = await instance.schema.hasTable('crawl_errors');
-    if (hasTable) {
-        return;
-    }
-    const hasPages = await instance.schema.hasTable('pages');
-    if (!hasPages) {
-        // Empty archive; the regular initSchema path will create the table.
-        return;
-    }
-    await instance.schema.createTable('crawl_errors', (t) => {
-        t.increments('id');
-        t.string('url', 8190).nullable();
-        t.boolean('isExternal');
-        t.text('message').notNullable();
-        t.integer('createdAt').notNullable();
-    });
-    // eslint-disable-next-line no-console
-    console.error('[migrate] crawl_errors table created');
-}

package/lib/archive/migrate-html-blob-tables.d.ts

@@ -1,24 +0,0 @@
-import type { Knex } from 'knex';
-/**
- * Adds the BLOB-backed HTML snapshot tables (`page_html_blobs` +
- * `page_html_ref`) to archives created before #75.
- *
- * A pre-#75 `.nitpicker` already has the `info` table, so `initSchema`'s
- * early-return path skips the freshly-introduced `CREATE TABLE`s. This
- * migration brings the schema forward idempotently so the read API stops
- * raising raw "no such table" errors against legacy archives. It does NOT
- * touch the `pages.html` column or backfill the new tables from the
- * archive's `snapshot-html.zip`: that data migration belongs to the
- * `scripts/migrate-to-0.10.mjs` one-shot, which can run whenever the
- * user is ready to commit the (multi-hour) CPU cost.
- *
- * Outcome on a legacy archive that has NOT yet been data-migrated:
- *   - new tables exist but are empty,
- *   - `getHtmlOfPageById` returns `null` for every page (no row in `page_html_ref`),
- *   - viewer / MCP / analyze plugins see "snapshot unavailable" instead of crashing.
- *
- * The migration prints a one-line notice when it runs so the user knows
- * the archive was upgraded in place.
- * @param instance - The Knex query builder instance connected to the database.
- */
-export declare function migrateHtmlBlobTables(instance: Knex): Promise<void>;

package/lib/archive/migrate-html-blob-tables.js

@@ -1,53 +0,0 @@
-/**
- * Adds the BLOB-backed HTML snapshot tables (`page_html_blobs` +
- * `page_html_ref`) to archives created before #75.
- *
- * A pre-#75 `.nitpicker` already has the `info` table, so `initSchema`'s
- * early-return path skips the freshly-introduced `CREATE TABLE`s. This
- * migration brings the schema forward idempotently so the read API stops
- * raising raw "no such table" errors against legacy archives. It does NOT
- * touch the `pages.html` column or backfill the new tables from the
- * archive's `snapshot-html.zip`: that data migration belongs to the
- * `scripts/migrate-to-0.10.mjs` one-shot, which can run whenever the
- * user is ready to commit the (multi-hour) CPU cost.
- *
- * Outcome on a legacy archive that has NOT yet been data-migrated:
- *   - new tables exist but are empty,
- *   - `getHtmlOfPageById` returns `null` for every page (no row in `page_html_ref`),
- *   - viewer / MCP / analyze plugins see "snapshot unavailable" instead of crashing.
- *
- * The migration prints a one-line notice when it runs so the user knows
- * the archive was upgraded in place.
- * @param instance - The Knex query builder instance connected to the database.
- */
-export async function migrateHtmlBlobTables(instance) {
-    const hasBlobs = await instance.schema.hasTable('page_html_blobs');
-    if (hasBlobs) {
-        return;
-    }
-    const hasPages = await instance.schema.hasTable('pages');
-    if (!hasPages) {
-        // Empty archive; initSchema will create both tables on the
-        // fresh-DB path. Nothing to migrate.
-        return;
-    }
-    await instance.raw(`
-		CREATE TABLE page_html_blobs (
-			hash         BLOB PRIMARY KEY,
-			body         BLOB NOT NULL,
-			codec        TEXT NOT NULL CHECK(codec IN ('zstd', 'none')),
-			size_raw     INTEGER NOT NULL,
-			size_stored  INTEGER NOT NULL
-		) WITHOUT ROWID
-	`);
-    await instance.raw(`
-		CREATE TABLE page_html_ref (
-			page_id  INTEGER PRIMARY KEY REFERENCES pages(id) ON DELETE CASCADE,
-			hash     BLOB NOT NULL REFERENCES page_html_blobs(hash)
-		) WITHOUT ROWID
-	`);
-    await instance.raw('CREATE INDEX idx_page_html_ref_hash ON page_html_ref(hash)');
-    // eslint-disable-next-line no-console
-    console.error('[migrate] page_html_blobs / page_html_ref tables created. ' +
-        'HTML snapshots are empty until `node scripts/migrate-to-0.10.mjs` is run on the original archive.');
-}

package/lib/archive/migrate-page-errors.d.ts

@@ -1,16 +0,0 @@
-import type { Knex } from 'knex';
-/**
- * Adds the `page_errors` table to archives created before partial-failure
- * recording landed.
- *
- * `page_errors` captures secondary scrape failures (e.g. a viewport switch
- * that detaches the frame and surfaces `retryExhausted`) so they show up in
- * the archive instead of being lost to stdout logs. Older `.nitpicker` files
- * predate this table; this migration creates it idempotently.
- *
- * Idempotent: when the table already exists, the function exits without
- * touching the schema or writing to stderr. When it has to run, a single
- * notice is written so the user knows the file was upgraded.
- * @param instance - The Knex query builder instance connected to the database.
- */
-export declare function migratePageErrors(instance: Knex): Promise<void>;