@nitpicker/crawler 0.9.0 → 0.12.0

package/lib/crawler.d.ts

@@ -7,8 +7,6 @@
 export * from './utils/types/types.js';
 export { eachSplitted } from './utils/array/each-splitted.js';
 export { DOMEvaluationError } from './utils/error/dom-evaluation-error.js';
-export { ErrorEmitter } from './utils/error/error-emitter.js';
-export type { ErrorEvent } from './utils/error/error-emitter.js';
 export * from './utils/object/clean-object.js';
 export { globalLog as log } from './utils/debug.js';
 export { ArchiveAccessor } from './archive/archive-accessor.js';
@@ -17,6 +15,14 @@ export { default as Page } from './archive/page.js';
 export { default as ArchiveResource } from './archive/resource.js';
 export * from './archive/types.js';
 export { default as Archive } from './archive/archive.js';
+export { isArchiveCacheDisabled } from './archive/cache/is-archive-cache-disabled.js';
+export { peekArchiveLockHolder } from './archive/peek-archive-lock.js';
+export type { ArchiveLockHolder } from './archive/peek-archive-lock.js';
+export type { FlatPageMetaColumns, JsonLdRow, JsonLdRowForInsert, TagRow, TagRowForInsert, JsonLdSummary, TagsSummary, TagInventoryEntry, PageDenormalizedColumns, } from './archive/meta/types.js';
+export { IncompatibleArchiveError } from './archive/meta/types.js';
+export { REQUIRED_FORMAT_VERSION } from './archive/meta/assert-compatible-version.js';
 export { DEFAULT_EXCLUDED_EXTERNAL_URLS, CrawlerOrchestrator, } from './crawler-orchestrator.js';
 export * from './types.js';
 export * from './crawler/types.js';
+export { classifyErrorKind } from './classify-error-kind.js';
+export { computeFileSha256 } from './utils/compute-file-sha256.js';

package/lib/crawler.js

@@ -8,7 +8,6 @@
 export * from './utils/types/types.js';
 export { eachSplitted } from './utils/array/each-splitted.js';
 export { DOMEvaluationError } from './utils/error/dom-evaluation-error.js';
-export { ErrorEmitter } from './utils/error/error-emitter.js';
 export * from './utils/object/clean-object.js';
 export { globalLog as log } from './utils/debug.js';
 // Archive
@@ -17,7 +16,13 @@ export { default as Page } from './archive/page.js';
 export { default as ArchiveResource } from './archive/resource.js';
 export * from './archive/types.js';
 export { default as Archive } from './archive/archive.js';
+export { isArchiveCacheDisabled } from './archive/cache/is-archive-cache-disabled.js';
+export { peekArchiveLockHolder } from './archive/peek-archive-lock.js';
+export { IncompatibleArchiveError } from './archive/meta/types.js';
+export { REQUIRED_FORMAT_VERSION } from './archive/meta/assert-compatible-version.js';
 // Core
 export { DEFAULT_EXCLUDED_EXTERNAL_URLS, CrawlerOrchestrator, } from './crawler-orchestrator.js';
 export * from './types.js';
 export * from './crawler/types.js';
+export { classifyErrorKind } from './classify-error-kind.js';
+export { computeFileSha256 } from './utils/compute-file-sha256.js';

package/lib/permanent-error-kinds.d.ts

@@ -0,0 +1,43 @@
+import type { ErrorKind } from './types.js';
+/**
+ * Error kinds whose failure mode is **persistent** — re-fetching the same URL
+ * (without changing the network, the certificate, the destination host, or the
+ * browser's block-list state) will almost certainly produce the same failure.
+ *
+ * Used by {@link resetFailedPages} to exclude pages whose latest recorded error
+ * falls in this set, so `--retry-failed` actually converges: without the
+ * exclusion, NXDOMAIN / TLS mismatch / `ERR_BLOCKED_BY_CLIENT` /
+ * `ECONNREFUSED` / HTTP parse-error pages would be reset to pending on every
+ * iteration, the crawler would re-attempt them, they would fail again the
+ * same way, and the retry-target count would stay constant forever.
+ *
+ * Why these five and not others:
+ * - **dns** — `ENOTFOUND` / `ERR_NAME_NOT_RESOLVED` are authoritative DNS
+ *   answers; the host is gone (or never existed). EAI_AGAIN is split out as
+ *   `dns-transient` precisely so it is NOT in this set.
+ * - **tls** — cert expiry / SAN mismatch / SSL protocol errors require the
+ *   server operator to fix something; retrying within the same archive run
+ *   cannot succeed.
+ * - **client-blocked** — Chromium's `ERR_BLOCKED_*` family is a deliberate
+ *   browser-side rejection; the request will be blocked identically every
+ *   time the browser sees the same URL.
+ * - **parse-error** — the server's HTTP response is malformed (`Expected
+ *   HTTP/, RTSP/ or ICE/`, `Unexpected end of stream`). Retrying the same
+ *   request hits the same parser failure.
+ * - **connection-refused** — `ECONNREFUSED` is an authoritative TCP RST from
+ *   the listener; either no process is listening on the port or its accept
+ *   queue rejected the connection. Either way the answer is final until the
+ *   server operator intervenes.
+ *
+ * Notably absent (intentionally retryable):
+ * - `connection-reset` / `connection-timeout` — could be middlebox or
+ *   transient overload
+ * - `dns-transient` (EAI_AGAIN / EREFUSED) — local resolver hiccup
+ * - `local-network` — operator-side connectivity loss
+ * - `timeout` — slow but reachable server (HEAD-timeout escalation gives
+ *   these a real chance)
+ * - `protocol` — puppeteer lifecycle race, often recovers on retry
+ * - `unknown` — by definition we don't know it's permanent, so we keep
+ *   retrying (errs on the side of investigating)
+ */
+export declare const PERMANENT_ERROR_KINDS: ReadonlySet<ErrorKind>;

package/lib/permanent-error-kinds.js

@@ -0,0 +1,48 @@
+/**
+ * Error kinds whose failure mode is **persistent** — re-fetching the same URL
+ * (without changing the network, the certificate, the destination host, or the
+ * browser's block-list state) will almost certainly produce the same failure.
+ *
+ * Used by {@link resetFailedPages} to exclude pages whose latest recorded error
+ * falls in this set, so `--retry-failed` actually converges: without the
+ * exclusion, NXDOMAIN / TLS mismatch / `ERR_BLOCKED_BY_CLIENT` /
+ * `ECONNREFUSED` / HTTP parse-error pages would be reset to pending on every
+ * iteration, the crawler would re-attempt them, they would fail again the
+ * same way, and the retry-target count would stay constant forever.
+ *
+ * Why these five and not others:
+ * - **dns** — `ENOTFOUND` / `ERR_NAME_NOT_RESOLVED` are authoritative DNS
+ *   answers; the host is gone (or never existed). EAI_AGAIN is split out as
+ *   `dns-transient` precisely so it is NOT in this set.
+ * - **tls** — cert expiry / SAN mismatch / SSL protocol errors require the
+ *   server operator to fix something; retrying within the same archive run
+ *   cannot succeed.
+ * - **client-blocked** — Chromium's `ERR_BLOCKED_*` family is a deliberate
+ *   browser-side rejection; the request will be blocked identically every
+ *   time the browser sees the same URL.
+ * - **parse-error** — the server's HTTP response is malformed (`Expected
+ *   HTTP/, RTSP/ or ICE/`, `Unexpected end of stream`). Retrying the same
+ *   request hits the same parser failure.
+ * - **connection-refused** — `ECONNREFUSED` is an authoritative TCP RST from
+ *   the listener; either no process is listening on the port or its accept
+ *   queue rejected the connection. Either way the answer is final until the
+ *   server operator intervenes.
+ *
+ * Notably absent (intentionally retryable):
+ * - `connection-reset` / `connection-timeout` — could be middlebox or
+ *   transient overload
+ * - `dns-transient` (EAI_AGAIN / EREFUSED) — local resolver hiccup
+ * - `local-network` — operator-side connectivity loss
+ * - `timeout` — slow but reachable server (HEAD-timeout escalation gives
+ *   these a real chance)
+ * - `protocol` — puppeteer lifecycle race, often recovers on retry
+ * - `unknown` — by definition we don't know it's permanent, so we keep
+ *   retrying (errs on the side of investigating)
+ */
+export const PERMANENT_ERROR_KINDS = new Set([
+    'dns',
+    'tls',
+    'client-blocked',
+    'parse-error',
+    'connection-refused',
+]);

package/lib/resource-row-to-lookup-result.d.ts

@@ -0,0 +1,13 @@
+import type { DB_Resource } from './archive/types.js';
+import type { ResourceLookupResult } from './crawler/types.js';
+/**
+ * Convert a raw `resources` table row into the minimal lookup result the
+ * crawler needs to reuse captured sub-resource data.
+ *
+ * Header parsing degrades to `null` on malformed JSON instead of throwing
+ * because a missing header set only loses fidelity — the reuse path stays
+ * valid.
+ * @param row - The raw database row.
+ * @returns The lookup result consumed by the crawler's resource-reuse hook.
+ */
+export declare function resourceRowToLookupResult(row: DB_Resource): ResourceLookupResult;

package/lib/resource-row-to-lookup-result.js

@@ -0,0 +1,20 @@
+import { parseResponseHeaders } from './utils/object/parse-response-headers.js';
+/**
+ * Convert a raw `resources` table row into the minimal lookup result the
+ * crawler needs to reuse captured sub-resource data.
+ *
+ * Header parsing degrades to `null` on malformed JSON instead of throwing
+ * because a missing header set only loses fidelity — the reuse path stays
+ * valid.
+ * @param row - The raw database row.
+ * @returns The lookup result consumed by the crawler's resource-reuse hook.
+ */
+export function resourceRowToLookupResult(row) {
+    return {
+        status: row.status,
+        statusText: row.statusText,
+        contentType: row.contentType,
+        contentLength: row.contentLength,
+        responseHeaders: parseResponseHeaders(row.responseHeaders),
+    };
+}

package/lib/types.d.ts

@@ -1,4 +1,88 @@
-import type { CrawlerError } from './utils/types/types.js';
+import type { CrawlerError, PageData } from './utils/types/types.js';
+/**
+ * Aggregate counts captured during a `--inventory` invocation, forwarded to
+ * `#writeInventoryRunRow` so the audit log row is consistent between the
+ * HTML-seed branch and the non-HTML-only branch of
+ * {@link CrawlerOrchestrator.inventory}.
+ *
+ * Spelled out here (not inlined at the call site) so a new field added to
+ * the audit row has a single edit point and so each field's semantics are
+ * documented per-property rather than scattered across the two emit sites.
+ */
+export interface InventoryRunAggregates {
+    /** Total non-empty lines in the input list (= `inventoryUrls.length` before any filtering). Stored verbatim as `inventory_runs.total_lines`. */
+    inventoryUrlsCount: number;
+    /** Number of novel URLs classified as HTML and queued for render. Stored as `new_pages` (excludes anchor-discovered descendants — those add later via the crawler graph and are NOT counted here). */
+    htmlSeedsCount: number;
+    /** Number of novel URLs classified as non-HTML and written directly into `resources`. Stored as `new_resources`. */
+    nonHtmlCount: number;
+    /** URLs dropped because they fell outside the archived scope. Stored as `scope_skipped`. */
+    outOfScope: number;
+    /**
+     * SHA-256 hex digest of the source `.txt`, **pre-computed by the caller**
+     * (typically the CLI's `inventoryCrawl`). Stored verbatim as
+     * `inventory_runs.source_file_sha256`.
+     *
+     * Pre-computation lifts the absolute path off the orchestrator
+     * boundary entirely — the path is privacy-sensitive (leaks
+     * user-home / OS structure when archives are shared), and the
+     * orchestrator has no business handling it after the audit-row
+     * column was dropped. Pass `null` for programmatic callers that
+     * built `inventoryUrls` in-memory; the audit row's
+     * `source_file_sha256` will be `NULL`.
+     */
+    sourceFileSha256: string | null;
+}
+/**
+ * Coarse cause of a crawl/scrape failure.
+ *
+ * The crawler stores only the raw error message (in `crawl_errors`,
+ * `page_errors`, or `error.log`); the cause is derived on read by
+ * `classifyErrorKind`, so existing archives gain classification without a
+ * re-crawl.
+ *
+ * Owned by the crawler package because both the crawler (for DNS-burned host
+ * caching) and `@nitpicker/query` (for `getErrorKinds` / `getSummary`) need to
+ * classify error messages, and crawler cannot depend on query.
+ *
+ * ### transient vs persistent
+ *
+ * | kind | transient? | DNS-burn? | notes |
+ * | --- | --- | --- | --- |
+ * | `dns` | no | yes | NXDOMAIN; the host does not resolve at all |
+ * | `dns-transient` | **yes** | no | `EAI_AGAIN`; local resolver hiccup, retry often recovers |
+ * | `tls` | no | no | certificate issue, usually persistent until cert rotates |
+ * | `connection-refused` | mostly persistent | no | server actively rejecting on this port |
+ * | `connection-reset` | yes | no | TCP reset mid-stream, often transient |
+ * | `connection-timeout` | yes | no | TCP-level timeout (`ETIMEDOUT`); slow but reachable |
+ * | `local-network` | **yes** | no | local machine's network is unreachable / changed (WiFi, sleep, ICMP-unreachable, …) |
+ * | `parse-error` | mostly persistent | no | HTTP response could not be parsed (proxy, garbage, MITM) |
+ * | `client-blocked` | persistent (per browser) | no | Chromium-side `ERR_BLOCKED_BY_*` family — the browser actively refused the request (ad/tracker heuristics, CSP, CORP, administrator block list, …) |
+ * | `protocol` | yes | no | puppeteer protocol layer (frame detached, target closed, …) |
+ * | `timeout` | yes | no | puppeteer navigation timeout or HEAD pre-flight race timeout (`Timeout: <url>`) |
+ * | `unknown` | unknown | no | catch-all for messages no matcher recognised |
+ *
+ * Only `dns` is mark-target for the DNS-burned host cache; everything else is
+ * either too transient to burn (network glitch / browser hiccup) or too
+ * server-specific to extrapolate to "this whole host is dead."
+ *
+ * ### Derived constants that MUST be reviewed when this union changes
+ *
+ * - `PERMANENT_ERROR_KINDS` (`permanent-error-kinds.ts`) — the set of kinds
+ *   excluded from `--retry-failed` so retry iterations actually converge.
+ *   A new kind that is deterministically permanent (server-state, cert,
+ *   browser-block, …) likely belongs here.
+ * - `PUPPETEER_FALLBACK_KINDS` (`crawler/is-puppeteer-fallback-candidate.ts`)
+ *   — the set of kinds where one puppeteer attempt has a realistic chance
+ *   of succeeding after HEAD+GET pre-flight exhausted retries. A new kind
+ *   modelling a middlebox / WAF / slow-server quirk likely belongs here.
+ *
+ * Adding a kind without reviewing both sets risks a silent regression:
+ * `--retry-failed` re-trying a permanent failure forever (no PERMANENT
+ * entry), or a recoverable URL never reaching the puppeteer fallback (no
+ * PUPPETEER_FALLBACK entry).
+ */
+export type ErrorKind = 'dns' | 'dns-transient' | 'connection-refused' | 'connection-reset' | 'connection-timeout' | 'tls' | 'local-network' | 'parse-error' | 'client-blocked' | 'timeout' | 'protocol' | 'unknown';
 /**
  * Event map for the `CrawlerOrchestrator` class.
  *
@@ -24,4 +108,14 @@ export interface CrawlEvent {
      * Emitted when an error occurs during crawling or archiving.
      */
     error: CrawlerError;
+    /**
+     * Emitted when a URL redirects to a destination already rendered during this
+     * crawl, so only the redirect edge is recorded and the destination is not
+     * re-rendered (#73). Mirrors the crawler's `redirect` event; useful for
+     * observing how much redirect-convergence work was skipped.
+     */
+    redirect: {
+        /** HEAD-resolved page data carrying the redirect chain (source → destination). */
+        result: PageData;
+    };
 }

package/lib/utils/compute-file-sha256.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Compute the SHA-256 digest of a file's bytes, streaming so memory stays
+ * O(1) regardless of file size.
+ *
+ * Used by {@link CrawlerOrchestrator.inventory} to fingerprint the source
+ * URL list `.txt` and store the digest on the `inventory_runs` row.
+ * Phase 3 (`--refresh`) will key dedupe on this column; Phase 1 just
+ * records it for operator audit.
+ *
+ * Returns `null` instead of throwing when the file cannot be read (e.g.
+ * vanished mid-run, permissions issue) so the inventory run's audit row
+ * can still be written with a NULL digest. Hashing failure is an audit
+ * loss, not a correctness failure — the actual ingestion has already
+ * succeeded by the time this is called.
+ * @param filePath - Absolute or relative path to the file to hash.
+ * @returns Lower-case hex digest (64 chars), or `null` if reading failed.
+ * @example
+ * ```ts
+ * const sha = await computeFileSha256('/tmp/list.txt');
+ * if (sha) console.log(sha); // 64-char hex string
+ * ```
+ */
+export declare function computeFileSha256(filePath: string): Promise<string | null>;

package/lib/utils/compute-file-sha256.js

@@ -0,0 +1,55 @@
+import { createHash } from 'node:crypto';
+import { createReadStream } from 'node:fs';
+import { crawlerLog } from '../debug.js';
+/**
+ * Compute the SHA-256 digest of a file's bytes, streaming so memory stays
+ * O(1) regardless of file size.
+ *
+ * Used by {@link CrawlerOrchestrator.inventory} to fingerprint the source
+ * URL list `.txt` and store the digest on the `inventory_runs` row.
+ * Phase 3 (`--refresh`) will key dedupe on this column; Phase 1 just
+ * records it for operator audit.
+ *
+ * Returns `null` instead of throwing when the file cannot be read (e.g.
+ * vanished mid-run, permissions issue) so the inventory run's audit row
+ * can still be written with a NULL digest. Hashing failure is an audit
+ * loss, not a correctness failure — the actual ingestion has already
+ * succeeded by the time this is called.
+ * @param filePath - Absolute or relative path to the file to hash.
+ * @returns Lower-case hex digest (64 chars), or `null` if reading failed.
+ * @example
+ * ```ts
+ * const sha = await computeFileSha256('/tmp/list.txt');
+ * if (sha) console.log(sha); // 64-char hex string
+ * ```
+ */
+export async function computeFileSha256(filePath) {
+    return await new Promise((resolve) => {
+        const hash = createHash('sha256');
+        const stream = createReadStream(filePath);
+        let settled = false;
+        const settle = (value) => {
+            if (settled) {
+                return;
+            }
+            settled = true;
+            resolve(value);
+        };
+        stream.on('data', (chunk) => hash.update(chunk));
+        stream.on('end', () => settle(hash.digest('hex')));
+        stream.on('error', (error) => {
+            // Trace the failure so operators inspecting `--inventory`
+            // audit rows with `source_file_sha256 = NULL` can recover
+            // the underlying cause via `DEBUG=Nitpicker:Crawler:*`.
+            crawlerLog('compute-file-sha256 failed for %s: %s', filePath, error.message);
+            settle(null);
+        });
+        // `'close'` is the last-resort settler. If the stream is
+        // destroyed externally between `'data'` and `'end'` (e.g. a
+        // test or signal handler), neither `'end'` nor `'error'` may
+        // fire — leaving the promise pending forever. Falling back to
+        // `null` matches the documented contract ("hashing failure
+        // yields null").
+        stream.on('close', () => settle(null));
+    });
+}

package/lib/utils/error/emit-error-with-retry.d.ts

@@ -0,0 +1,40 @@
+import type { RetryCallOptions } from '@d-zero/shared/retry';
+/**
+ * Minimal object shape required by {@link emitErrorAndRetry}: an emitter
+ * able to publish an `'error'` event carrying an `Error` payload. Same
+ * contract as `emitError` — see {@link ./emit-error.js} for details.
+ */
+interface Emitter {
+    /**
+     * Emits the `'error'` event with the given `Error` payload.
+     * @param event - Event name; always `'error'` here.
+     * @param payload - The error instance to publish.
+     * @returns Anything; the caller ignores it.
+     */
+    emit(event: 'error', payload: Error): unknown;
+}
+/**
+ * Retries `fn` per `retryOptions`, then — if retries are exhausted with a
+ * final `Error` — logs and re-emits it as an `'error'` event on `emitter`
+ * before re-throwing. `label` is forwarded into `retryCall` so its own
+ * onWait / onGiveUp / timeout messages identify the call site (previously
+ * derived automatically by the `@retry` decorator via
+ * `ClassMethodDecoratorContext.name`).
+ *
+ * Inlined try/catch rather than composing {@link emitError} with a
+ * `() => retryCall(...)` closure to keep the per-invocation closure count
+ * at 1 — this method is called on the DB write hot path (`updatePage`,
+ * `insertResource`), where wrapping decorators previously ran with zero
+ * per-call allocation.
+ * @template T - The return type of `fn`.
+ * @param emitter - The event emitter that receives the `'error'` event.
+ * @param label - Human-readable label; forwarded into `retryCall`
+ *                (visible in retry timeouts / onWait / onGiveUp) and used
+ *                as the debug log prefix on final failure.
+ * @param fn - The async operation to retry.
+ * @param retryOptions - Retry configuration; `label` is overwritten
+ *                       with the `label` argument here.
+ * @returns The resolved value of `fn`.
+ */
+export declare function emitErrorAndRetry<T>(emitter: Emitter, label: string, fn: () => Promise<T>, retryOptions: RetryCallOptions): Promise<T>;
+export {};

package/lib/utils/error/emit-error-with-retry.js

@@ -0,0 +1,44 @@
+import { retryCall } from '@d-zero/shared/retry';
+import { log } from '../debug.js';
+/**
+ * Debug logger for the ErrorEmitter helper family.
+ * Namespace: `Nitpicker:Utils:ErrorEmitter`. Kept identical to the namespace
+ * used by the deleted `@ErrorEmitter()` decorator so existing `DEBUG` env
+ * filters keep working after the decorator-to-HOF migration.
+ */
+const errorLog = log.extend('ErrorEmitter');
+/**
+ * Retries `fn` per `retryOptions`, then — if retries are exhausted with a
+ * final `Error` — logs and re-emits it as an `'error'` event on `emitter`
+ * before re-throwing. `label` is forwarded into `retryCall` so its own
+ * onWait / onGiveUp / timeout messages identify the call site (previously
+ * derived automatically by the `@retry` decorator via
+ * `ClassMethodDecoratorContext.name`).
+ *
+ * Inlined try/catch rather than composing {@link emitError} with a
+ * `() => retryCall(...)` closure to keep the per-invocation closure count
+ * at 1 — this method is called on the DB write hot path (`updatePage`,
+ * `insertResource`), where wrapping decorators previously ran with zero
+ * per-call allocation.
+ * @template T - The return type of `fn`.
+ * @param emitter - The event emitter that receives the `'error'` event.
+ * @param label - Human-readable label; forwarded into `retryCall`
+ *                (visible in retry timeouts / onWait / onGiveUp) and used
+ *                as the debug log prefix on final failure.
+ * @param fn - The async operation to retry.
+ * @param retryOptions - Retry configuration; `label` is overwritten
+ *                       with the `label` argument here.
+ * @returns The resolved value of `fn`.
+ */
+export async function emitErrorAndRetry(emitter, label, fn, retryOptions) {
+    try {
+        return await retryCall(fn, { ...retryOptions, label });
+    }
+    catch (error) {
+        if (error instanceof Error) {
+            errorLog('%s: %O', label, error);
+            void emitter.emit('error', error);
+        }
+        throw error;
+    }
+}

package/lib/utils/error/emit-error.d.ts

@@ -0,0 +1,39 @@
+/**
+ * Minimal object shape required by {@link emitError}: an emitter able to
+ * publish an `'error'` event carrying an `Error` payload. Any class that
+ * extends `TypedAwaitEventEmitter<{ error: Error, ... }>` satisfies this,
+ * and so does Node's built-in `EventEmitter`.
+ */
+interface Emitter {
+    /**
+     * Emits the `'error'` event with the given `Error` payload.
+     * The return value is intentionally unspecified — callers ignore it.
+     * @param event - Event name; always `'error'` here.
+     * @param payload - The error instance to publish.
+     * @returns Anything; the caller ignores it.
+     */
+    emit(event: 'error', payload: Error): unknown;
+}
+/**
+ * Wraps `fn` so that any thrown `Error` is logged with `label` and
+ * re-emitted as an `'error'` event on `emitter` before being re-thrown.
+ * Non-`Error` throws pass through unchanged — no log, no emit — matching
+ * the behaviour of the deleted `@ErrorEmitter()` decorator so that
+ * downstream `'error'` listeners (e.g. the crawler orchestrator that
+ * aborts on any emitted error) are only invoked with real `Error`
+ * instances.
+ *
+ * Kept as a standalone HOF (instead of a decorator) because Vite 8 /
+ * Vitest 4.1 switched from esbuild to Rolldown + oxc, which emit
+ * TC39 Stage 3 decorator syntax as-is and let it reach Node, causing
+ * `SyntaxError: Invalid or unexpected token` at test runtime.
+ * @template T - The return type of `fn`.
+ * @param emitter - The event emitter that receives the `'error'` event.
+ * @param label - Human-readable label included in the debug log line
+ *                (typically `'ClassName.methodName'`). Also useful as the
+ *                grep target when triaging failed crawls.
+ * @param fn - The async operation to wrap.
+ * @returns The resolved value of `fn`.
+ */
+export declare function emitError<T>(emitter: Emitter, label: string, fn: () => Promise<T>): Promise<T>;
+export {};

package/lib/utils/error/emit-error.js

@@ -0,0 +1,41 @@
+import { log } from '../debug.js';
+/**
+ * Debug logger for the ErrorEmitter helper family.
+ * Namespace: `Nitpicker:Utils:ErrorEmitter`. Kept identical to the namespace
+ * used by the deleted `@ErrorEmitter()` decorator so existing `DEBUG` env
+ * filters keep working after the decorator-to-HOF migration.
+ */
+const errorLog = log.extend('ErrorEmitter');
+/**
+ * Wraps `fn` so that any thrown `Error` is logged with `label` and
+ * re-emitted as an `'error'` event on `emitter` before being re-thrown.
+ * Non-`Error` throws pass through unchanged — no log, no emit — matching
+ * the behaviour of the deleted `@ErrorEmitter()` decorator so that
+ * downstream `'error'` listeners (e.g. the crawler orchestrator that
+ * aborts on any emitted error) are only invoked with real `Error`
+ * instances.
+ *
+ * Kept as a standalone HOF (instead of a decorator) because Vite 8 /
+ * Vitest 4.1 switched from esbuild to Rolldown + oxc, which emit
+ * TC39 Stage 3 decorator syntax as-is and let it reach Node, causing
+ * `SyntaxError: Invalid or unexpected token` at test runtime.
+ * @template T - The return type of `fn`.
+ * @param emitter - The event emitter that receives the `'error'` event.
+ * @param label - Human-readable label included in the debug log line
+ *                (typically `'ClassName.methodName'`). Also useful as the
+ *                grep target when triaging failed crawls.
+ * @param fn - The async operation to wrap.
+ * @returns The resolved value of `fn`.
+ */
+export async function emitError(emitter, label, fn) {
+    try {
+        return await fn();
+    }
+    catch (error) {
+        if (error instanceof Error) {
+            errorLog('%s: %O', label, error);
+            void emitter.emit('error', error);
+        }
+        throw error;
+    }
+}

package/lib/utils/object/parse-response-headers.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Parse a JSON-serialized HTTP response headers column from the archive
+ * database.
+ *
+ * The columns hold JSON produced by `JSON.stringify` at insert time; absent,
+ * malformed, or non-object JSON (the string `"null"`, arrays, primitives)
+ * degrades to `null` instead of throwing because a missing header set only
+ * loses fidelity — callers decide their own fallback (`?? {}` etc.).
+ * @param json - The raw column value.
+ * @returns The parsed header record, or `null` when absent or malformed.
+ */
+export declare function parseResponseHeaders(json: string | null): Record<string, string | string[] | undefined> | null;

package/lib/utils/object/parse-response-headers.js

@@ -0,0 +1,26 @@
+/**
+ * Parse a JSON-serialized HTTP response headers column from the archive
+ * database.
+ *
+ * The columns hold JSON produced by `JSON.stringify` at insert time; absent,
+ * malformed, or non-object JSON (the string `"null"`, arrays, primitives)
+ * degrades to `null` instead of throwing because a missing header set only
+ * loses fidelity — callers decide their own fallback (`?? {}` etc.).
+ * @param json - The raw column value.
+ * @returns The parsed header record, or `null` when absent or malformed.
+ */
+export function parseResponseHeaders(json) {
+    if (json == null) {
+        return null;
+    }
+    try {
+        const parsed = JSON.parse(json);
+        if (parsed == null || typeof parsed !== 'object' || Array.isArray(parsed)) {
+            return null;
+        }
+        return parsed;
+    }
+    catch {
+        return null;
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@nitpicker/crawler",
-	"version": "0.9.0",
+	"version": "0.12.0",
 	"description": "Web crawler engine with headless browser rendering and archive storage",
 	"author": "D-ZERO",
 	"license": "Apache-2.0",
@@ -27,26 +27,26 @@
 		"clean": "tsc --build --clean"
 	},
 	"dependencies": {
-		"@d-zero/beholder": "2.1.2",
-		"@d-zero/dealer": "1.7.0",
-		"@d-zero/fs": "0.2.2",
-		"@d-zero/shared": "0.20.1",
+		"@d-zero/beholder": "3.1.3",
+		"@d-zero/dealer": "1.9.3",
+		"@d-zero/fs": "0.2.5",
+		"@d-zero/shared": "0.22.1",
 		"ansi-colors": "4.1.3",
 		"debug": "4.4.3",
-		"follow-redirects": "1.15.11",
-		"fs-extra": "11.3.3",
-		"knex": "3.1.0",
+		"follow-redirects": "1.16.0",
+		"fs-extra": "11.3.5",
+		"knex": "3.2.10",
 		"libsql": "0.5.29",
 		"puppeteer": "24.37.5",
 		"robots-parser": "3.0.1",
-		"tar": "7.5.9"
+		"tar": "7.5.16"
 	},
 	"devDependencies": {
-		"@types/debug": "4.1.12",
+		"@types/debug": "4.1.13",
 		"@types/follow-redirects": "1.14.4",
 		"@types/fs-extra": "11.0.4",
 		"@types/tar": "7.0.87",
 		"@types/unzipper": "0.10.11"
 	},
-	"gitHead": "2d194c881f323de7555e0025f6291ded65450dae"
+	"gitHead": "58b68416e0b6e3dcdd92772bee754f186a35505f"
 }

package/lib/utils/error/error-emitter.d.ts

@@ -1,18 +0,0 @@
-import type { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
-/**
- * Event payload type for error events emitted by classes using the {@link ErrorEmitter} decorator.
- * @template E - The specific error type, defaults to `Error`.
- */
-export type ErrorEvent<E extends Error = Error> = {
-    /** The error instance that was caught. */
-    error: E;
-};
-/**
- * A class method decorator factory that wraps the decorated method with error handling.
- * When the method throws an `Error`, it emits an `'error'` event on the class instance
- * (which must extend {@link EventEmitter}) with the caught error, then re-throws the error.
- * @template C - The class type, which must be an EventEmitter capable of emitting error events.
- * @template E - The error event type, defaults to {@link ErrorEvent}.
- * @returns A decorator function that wraps the target method with error-emitting behavior.
- */
-export declare function ErrorEmitter<C extends EventEmitter<E>, E extends ErrorEvent = ErrorEvent>(): (method: Function, context: ClassMethodDecoratorContext) => (this: C, ...args: unknown[]) => Promise<any>;