@nitpicker/crawler 0.8.0 → 0.11.0

package/lib/crawler-orchestrator.js

@@ -7,9 +7,13 @@ import pkg from '../package.json' with { type: 'json' };
 import Archive from './archive/archive.js';
 import { clearDestinationCache } from './crawler/clear-destination-cache.js';
 import Crawler from './crawler/crawler.js';
+import { fetchDestination } from './crawler/fetch-destination.js';
+import { findScopeEntry } from './crawler/find-scope-entry.js';
+import { isHtmlContentType } from './crawler/is-html-content-type.js';
 import { crawlerLog, log } from './debug.js';
 import { normalizeToArray } from './normalize-to-array.js';
 import { resolveOutputPath } from './resolve-output-path.js';
+import { resourceRowToLookupResult } from './resource-row-to-lookup-result.js';
 import { cleanObject } from './utils/object/clean-object.js';
 import { WriteQueue } from './write-queue.js';
 /**
@@ -53,6 +57,8 @@ export class CrawlerOrchestrator extends EventEmitter {
     #crawler;
     /** Whether the crawl was started from a pre-defined URL list (non-recursive mode). */
     #fromList;
+    /** Serializes archive writes from crawler event handlers (FIFO). */
+    #writeQueue = new WriteQueue();
     /**
      * The underlying archive instance used for storing crawl results.
      */
@@ -95,6 +101,26 @@ export class CrawlerOrchestrator extends EventEmitter {
             verbose: options?.verbose ?? false,
             userAgent: options?.userAgent || defaultUserAgent,
             ignoreRobots: options?.ignoreRobots ?? false,
+            // Let the crawler reuse sub-resource data captured during page
+            // rendering instead of issuing a redundant HEAD pre-flight.
+            lookupResource: async (urls) => {
+                // Fast path: read directly — the row is usually flushed long
+                // before the queued URL is dequeued, and a direct read does not
+                // block behind pending writes.
+                const direct = await this.#archive.getResourceByUrl(urls);
+                if (direct) {
+                    return resourceRowToLookupResult(direct);
+                }
+                // A miss may be an insert still queued — re-read serialized
+                // behind the write queue so hit/miss is deterministic.
+                const row = await this.#writeQueue.enqueue(() => this.#archive.getResourceByUrl(urls));
+                return row ? resourceRowToLookupResult(row) : null;
+            },
+            // Inventory mode is opted into by `CrawlerOrchestrator.inventory`
+            // (see T3); the default crawl path stays in normal mode so new
+            // rows continue to land in pages/resources with the DB DEFAULT
+            // `'crawled'` provenance label.
+            inventoryMode: options?.inventoryMode ?? null,
         });
     }
     /**
@@ -114,29 +140,35 @@ export class CrawlerOrchestrator extends EventEmitter {
      * when the crawl completes. Discovered pages, external pages, skipped pages,
      * and resources are forwarded to the archive for storage.
      * @param list - The list of parsed URLs to crawl. The first URL is used as the root.
+     * @param opts - Optional crawl overrides.
+     * @param opts.recursive - Whether discovered URLs are followed. Defaults to
+     *   `!fromList` (recursive unless the archive was created from a URL list), so
+     *   existing callers keep their behaviour; the retry flow passes it explicitly.
      * @returns A promise that resolves when crawling is complete.
      * @throws {Error} If the URL list is empty.
      */
-    async crawling(list) {
+    async crawling(list, opts) {
         const root = list[0];
         if (!root) {
             throw new Error('URL is empty');
         }
-        const writeQueue = new WriteQueue();
+        const writeQueue = this.#writeQueue;
         return new Promise((resolve, reject) => {
             this.#crawler.on('error', (error) => {
                 crawlerLog('On error: %O', error);
-                void writeQueue.enqueue(() => this.#archive.addError(error));
+                writeQueue
+                    .enqueue(() => this.#archive.addError(error))
+                    .catch((writeError) => reject(writeError));
                 void this.emit('error', error);
             });
-            this.#crawler.on('page', ({ result }) => {
+            this.#crawler.on('page', ({ result, source }) => {
                 writeQueue
-                    .enqueue(() => this.#archive.setPage(result))
+                    .enqueue(() => this.#archive.setPage(result, source))
                     .catch((error) => reject(error));
             });
-            this.#crawler.on('externalPage', ({ result }) => {
+            this.#crawler.on('externalPage', ({ result, source }) => {
                 writeQueue
-                    .enqueue(() => this.#archive.setExternalPage(result))
+                    .enqueue(() => this.#archive.setExternalPage(result, source))
                     .catch((error) => reject(error));
             });
             this.#crawler.on('skip', ({ url, reason, isExternal }) => {
@@ -144,9 +176,20 @@ export class CrawlerOrchestrator extends EventEmitter {
                     .enqueue(() => this.#archive.setSkippedPage(url, reason, isExternal))
                     .catch((error) => reject(error));
             });
-            this.#crawler.on('response', ({ resource }) => {
+            this.#crawler.on('pageError', ({ url, phase, message, isExternal }) => {
+                writeQueue
+                    .enqueue(() => this.#archive.addPageError(url, phase, message, isExternal))
+                    .catch((error) => reject(error));
+            });
+            this.#crawler.on('redirect', ({ result }) => {
+                writeQueue
+                    .enqueue(() => this.#archive.setRedirect(result))
+                    .catch((error) => reject(error));
+                void this.emit('redirect', { result });
+            });
+            this.#crawler.on('response', ({ resource, source }) => {
                 writeQueue
-                    .enqueue(() => this.#archive.setResources(resource))
+                    .enqueue(() => this.#archive.setResources(resource, source))
                     .catch((error) => reject(error));
             });
             this.#crawler.on('responseReferrers', (resource) => {
@@ -160,7 +203,7 @@ export class CrawlerOrchestrator extends EventEmitter {
                     .then(() => resolve())
                     .catch((error) => reject(error));
             });
-            this.#crawler.start(list, { recursive: !this.#fromList });
+            this.#crawler.start(list, { recursive: opts?.recursive ?? !this.#fromList });
         });
     }
     /**
@@ -348,7 +391,8 @@ export class CrawlerOrchestrator extends EventEmitter {
                 });
                 const { scraped, pending } = await archive.getCrawlingState();
                 const resources = await archive.getResourceUrlList();
-                orchestrator.#crawler.resume(pending, scraped, resources);
+                const pagesScrapedOffset = await archive.getScrapedHtmlPageCount();
+                orchestrator.#crawler.resume(pending, scraped, resources, pagesScrapedOffset);
                 if (initializedCallback) {
                     await initializedCallback(orchestrator, mergedConfig);
                 }
@@ -381,6 +425,338 @@ export class CrawlerOrchestrator extends EventEmitter {
             throw error;
         }
     }
+    /**
+     * Inventory mode: cross-reference a user-supplied URL list against an
+     * existing `.nitpicker` archive and import ONLY the URLs that are not yet
+     * tracked there. Designed to surface "orphan" landing pages that link
+     * graph traversal could not reach, and "unused" server-side files that
+     * no crawled page references — both of which the
+     * `listIsolatedPages` / `listUnusedResources` queries can then list.
+     *
+     * Flow:
+     *
+     * 1. Open the archive (writer mode, takes the archive lock).
+     * 2. Reject list-mode archives — they hold metadata-only rows that
+     *    inventory has no business touching.
+     * 3. Reject archives with unfinished `pending` URLs — those would inherit
+     *    the inventory `source` label by mistake. Operator must resume /
+     *    retry-failed first.
+     * 4. Parse the URL list. Anything outside the archived scope is warned
+     *    and skipped (inventory is per-server by design).
+     * 5. Subtract URLs that already exist in `pages` or `resources` so the
+     *    second (and N-th) inventory pass is a no-op for known rows — keeps
+     *    `'inventory-seed'` rows from being silently demoted.
+     * 6. Make `<archive>.bak`. Anything thrown beyond this point restores
+     *    from the backup.
+     * 7. HEAD-probe each novel URL. Responses classified as HTML are queued
+     *    as Crawler seeds (`'inventory-seed'`); everything else is recorded
+     *    in `resources` directly as `'inventory-seed'` (no browser launch).
+     * 8. If any HTML seeds exist, start a Crawler with
+     *    `inventoryMode = { seedUrls }` so the rendered page and every newly
+     *    discovered downstream link is labelled correctly. `resume` is fed
+     *    the existing `scraped` / `resources` sets so links into already-
+     *    crawled pages stop at the seen-gate without re-rendering.
+     * 9. Drop the backup on success; restore it on any throw.
+     *
+     * Mutually exclusive with `--append` / `--retry-failed` / `--resume` /
+     * `--diff` / `--list` / `--list-file` / `--single` / `--output` — the
+     * CLI dispatch enforces this; this method assumes the caller honoured
+     * the contract.
+     * @param archivePath - Absolute or cwd-relative path to the `.nitpicker` archive.
+     * @param inventoryUrls - Pre-read URL list (one URL per element).
+     * @param options - Optional config overrides — most callers leave this blank and let the archived config flow through.
+     * @param initializedCallback - Hook invoked once the orchestrator is constructed but before `crawling` runs (the CLI uses it to attach progress reporting).
+     * @returns The orchestrator instance after a successful inventory pass.
+     * @throws {Error} When `inventoryUrls` is empty, the archive is in list mode, or pending URLs from a previous crawl remain unresolved.
+     */
+    static async inventory(archivePath, inventoryUrls, options, initializedCallback) {
+        if (inventoryUrls.length === 0) {
+            throw new Error('inventory: URL list is empty');
+        }
+        const cwd = options?.cwd ?? process.cwd();
+        const absFilePath = path.isAbsolute(archivePath)
+            ? archivePath
+            : path.resolve(cwd, archivePath);
+        const archive = await Archive.open({ filePath: absFilePath, cwd });
+        try {
+            const archived = await archive.getConfig();
+            if (archived.fromList) {
+                throw new Error('Cannot run inventory on a list-mode archive: this archive was created with --list/--list-file and contains metadata-only pages. Create a fresh archive instead.');
+            }
+            const { scraped, pending } = await archive.getCrawlingState();
+            if (pending.length > 0) {
+                throw new Error(`inventory: archive has ${pending.length} pending URLs from a previous crawl. Resume or retry-failed first so inventory does not mislabel them as 'inventory-discovered'.`);
+            }
+            // Parse + scope-classify the candidate URLs. sortUrl drops
+            // unparseable strings; findScopeEntry separates in-scope from
+            // out-of-scope.
+            const parsedAll = sortUrl(inventoryUrls, archived);
+            const scopeMap = new Map();
+            for (const raw of archived.roots) {
+                const parsed = parseUrl(raw, archived);
+                if (!parsed)
+                    continue;
+                const existing = scopeMap.get(parsed.hostname) ?? [];
+                scopeMap.set(parsed.hostname, [...existing, parsed]);
+            }
+            const inScope = [];
+            let outOfScope = 0;
+            for (const url of parsedAll) {
+                if (findScopeEntry(url, scopeMap, archived) === null) {
+                    outOfScope++;
+                }
+                else {
+                    inScope.push(url);
+                }
+            }
+            if (outOfScope > 0) {
+                log('[inventory] %d URL(s) skipped (outside archived scope: %O)', outOfScope, archived.roots);
+            }
+            // Drop URLs that are already represented in the archive (either
+            // as pages or resources). Comparison key is `withoutHashAndAuth`
+            // to mirror what `#getIdByUrl` / `insertResource` actually store.
+            // Two independent reads — Promise.all halves the wait on large
+            // archives where each `WHERE url IN (?)` chunk costs real I/O.
+            const candidateUrls = inScope.map((u) => u.withoutHashAndAuth);
+            const [existingPageUrlList, existingResourceUrlList] = await Promise.all([
+                archive.getExistingPageUrls(candidateUrls),
+                archive.getExistingResourceUrls(candidateUrls),
+            ]);
+            const existingPageUrls = new Set(existingPageUrlList);
+            const existingResourceUrls = new Set(existingResourceUrlList);
+            const novelUrls = inScope.filter((u) => {
+                const key = u.withoutHashAndAuth;
+                return !existingPageUrls.has(key) && !existingResourceUrls.has(key);
+            });
+            const knownCount = existingPageUrls.size + existingResourceUrls.size;
+            log('[inventory] %d in-scope, %d already in archive, %d new', inScope.length, knownCount, novelUrls.length);
+            if (novelUrls.length === 0) {
+                // Nothing to do — release the archive cleanly without taking a
+                // backup. The orchestrator returned here is empty; the caller
+                // should only invoke `close` on it.
+                const noopConfig = {
+                    ...archived,
+                    ...cleanObject(options),
+                };
+                const orchestrator = new CrawlerOrchestrator(archive, noopConfig);
+                if (initializedCallback) {
+                    await initializedCallback(orchestrator, noopConfig);
+                }
+                return orchestrator;
+            }
+            const backupPath = absFilePath + '.bak';
+            await copyFile(absFilePath, backupPath);
+            try {
+                const headResults = await Promise.all(novelUrls.map(async (url) => {
+                    try {
+                        const head = await fetchDestination({
+                            url,
+                            isExternal: false,
+                            userAgent: archived.userAgent,
+                        });
+                        return { url, head, error: null };
+                    }
+                    catch (headError) {
+                        const error = headError instanceof Error ? headError : new Error(String(headError));
+                        return { url, head: null, error };
+                    }
+                }));
+                const htmlSeeds = [];
+                for (const result of headResults) {
+                    const { url, head, error } = result;
+                    if (error !== null) {
+                        // HEAD failure is recorded as a crawl_errors row so
+                        // the URL is visible in `query error-kinds`, but does
+                        // NOT abort the whole inventory pass — other novel
+                        // URLs may still succeed.
+                        await archive.addError({
+                            pid: process.pid,
+                            isMainProcess: true,
+                            url: url.href,
+                            isExternal: false,
+                            error,
+                        });
+                        continue;
+                    }
+                    if (head.contentType == null || isHtmlContentType(head.contentType)) {
+                        htmlSeeds.push(url);
+                    }
+                    else {
+                        await archive.setResources({
+                            url,
+                            isExternal: false,
+                            isError: false,
+                            status: head.status,
+                            statusText: head.statusText,
+                            contentType: head.contentType,
+                            contentLength: head.contentLength,
+                            compress: false,
+                            cdn: false,
+                            headers: head.responseHeaders ?? null,
+                        }, 'inventory-seed');
+                    }
+                }
+                // Config sent to the user-facing `initializedCallback`
+                // (matches the rest of the orchestrator's public surface —
+                // no inventory bookkeeping leaks out).
+                const baseConfig = {
+                    ...archived,
+                    ...cleanObject(options),
+                    recursive: true,
+                    fromList: false,
+                };
+                const seedSet = new Set(htmlSeeds.map((u) => u.withoutHashAndAuth));
+                // CrawlConfig overlay handed to the orchestrator constructor —
+                // carries the runtime-only `inventoryMode` that drives source
+                // labelling. Not persisted to the archive.
+                const orchestratorOptions = {
+                    ...baseConfig,
+                    inventoryMode: { seedUrls: seedSet },
+                };
+                if (htmlSeeds.length > 0) {
+                    const orchestrator = new CrawlerOrchestrator(archive, orchestratorOptions);
+                    const resources = await archive.getResourceUrlList();
+                    // Empty pending (we rejected non-empty above) but feed
+                    // every already-scraped URL into `seen` so the Crawler's
+                    // link enqueueing path drops links that hit a known
+                    // page without re-rendering it.
+                    orchestrator.#crawler.resume(pending, scraped, resources, 0);
+                    if (initializedCallback) {
+                        await initializedCallback(orchestrator, baseConfig);
+                    }
+                    log('Start inventory');
+                    log('Archive %s', absFilePath);
+                    log('HTML seeds %O', htmlSeeds.map((u) => u.href));
+                    await orchestrator.crawling(htmlSeeds, { recursive: true });
+                    clearDestinationCache();
+                    await archive.setUrlOrder();
+                    await ignoreEnoent(unlinkFile(backupPath));
+                    return orchestrator;
+                }
+                // Only non-HTML URLs were imported — nothing left to render,
+                // but still update sort order and finalize.
+                const orchestrator = new CrawlerOrchestrator(archive, orchestratorOptions);
+                if (initializedCallback) {
+                    await initializedCallback(orchestrator, baseConfig);
+                }
+                await archive.setUrlOrder();
+                await ignoreEnoent(unlinkFile(backupPath));
+                return orchestrator;
+            }
+            catch (error) {
+                try {
+                    await copyFile(backupPath, absFilePath);
+                    await ignoreEnoent(unlinkFile(backupPath));
+                }
+                catch (restoreError) {
+                    throw new AggregateError([error, restoreError], `inventory failed AND restore from backup failed. Original archive backup is left at: ${backupPath}`);
+                }
+                throw error;
+            }
+        }
+        catch (error) {
+            await archive.close().catch(() => { });
+            throw error;
+        }
+    }
+    /**
+     * Re-fetch previously-failed pages in an existing `.nitpicker` archive.
+     *
+     * Opens the archive, resets every page whose previous attempt ended in a
+     * recoverable failure (missing status / content type, or a 5xx status — see
+     * {@link Archive.resetFailedPages}) back to pending, and resumes crawling.
+     * The archived crawl configuration is reused — scopes, excludes, keywords,
+     * user agent, etc. — so the retry honours the original crawl boundaries
+     * unless a field is explicitly overridden via `options`. The exception is
+     * `recursive`: it is taken from `options` (the CLI flag defaults it to
+     * `true`) rather than inherited from the archive, so a retry decides afresh
+     * whether to follow newly-discovered URLs regardless of how the original
+     * crawl was run.
+     *
+     * When `recursive` is enabled (the default), newly-discovered URLs from the
+     * re-fetched pages are followed and crawled from scratch; when disabled, only
+     * the failed pages themselves are re-fetched. The archived roots seed the
+     * crawl scope while the reset pages are picked up through the resumed pending
+     * set, so failed external pages stay external (metadata-only) instead of being
+     * promoted into scope, and a failed root is re-fetched in place.
+     *
+     * A `<archive>.bak` is created before any DB mutation and removed on success;
+     * if the crawl throws, the backup is restored to keep the original archive
+     * intact.
+     *
+     * List-mode archives (`info.fromList === true`) are rejected for the same
+     * reason as {@link CrawlerOrchestrator.append}: their pages are metadata-only.
+     * @param archivePath - Absolute or relative path to the existing `.nitpicker`.
+     * @param options - Optional config overrides applied on top of the archived config.
+     * @param initializedCallback - Optional callback invoked after initialization but before crawling resumes.
+     * @returns The orchestrator instance after the retry crawl completes.
+     * @throws {Error} When the archive is in list mode or has no parseable roots.
+     */
+    static async retryFailed(archivePath, options, initializedCallback) {
+        const cwd = options?.cwd ?? process.cwd();
+        const absFilePath = path.isAbsolute(archivePath)
+            ? archivePath
+            : path.resolve(cwd, archivePath);
+        const archive = await Archive.open({ filePath: absFilePath, cwd });
+        // Any throw between here and the successful return must release the
+        // archive lock and clean up tmpDir; the caller's `close()` only runs on
+        // the happy path.
+        try {
+            const archived = await archive.getConfig();
+            if (archived.fromList) {
+                throw new Error('Cannot retry a list-mode archive: this archive was created with --list/--list-file and contains metadata-only pages. Create a fresh archive instead.');
+            }
+            const rootsParsed = sortUrl(archived.roots, archived);
+            if (rootsParsed.length === 0) {
+                throw new Error('retry: archive has no parseable root URLs');
+            }
+            const config = {
+                ...archived,
+                ...cleanObject(options),
+                roots: archived.roots,
+                fromList: false,
+                baseUrl: archived.baseUrl,
+            };
+            const backupPath = absFilePath + '.bak';
+            await copyFile(absFilePath, backupPath);
+            try {
+                const resetUrls = await archive.resetFailedPages();
+                log('Start retrying failed pages');
+                log('Archive %s', absFilePath);
+                log('Reset %d failed page(s)', resetUrls.length);
+                const orchestrator = new CrawlerOrchestrator(archive, config);
+                const { scraped, pending } = await archive.getCrawlingState();
+                const resources = await archive.getResourceUrlList();
+                const pagesScrapedOffset = await archive.getScrapedHtmlPageCount();
+                orchestrator.#crawler.resume(pending, scraped, resources, pagesScrapedOffset);
+                if (initializedCallback) {
+                    await initializedCallback(orchestrator, config);
+                }
+                await orchestrator.crawling(rootsParsed, { recursive: config.recursive });
+                clearDestinationCache();
+                await archive.setUrlOrder();
+                await ignoreEnoent(unlinkFile(backupPath));
+                return orchestrator;
+            }
+            catch (error) {
+                try {
+                    await copyFile(backupPath, absFilePath);
+                    await ignoreEnoent(unlinkFile(backupPath));
+                }
+                catch (restoreError) {
+                    // Restore itself failed — surface both so the operator knows
+                    // the .bak still exists and the original archive may be
+                    // corrupt. The outer `catch` still releases the lock.
+                    throw new AggregateError([error, restoreError], `retry failed AND restore from backup failed. Original archive backup is left at: ${backupPath}`);
+                }
+                throw error;
+            }
+        }
+        catch (error) {
+            await archive.close().catch(() => { });
+            throw error;
+        }
+    }
     /**
      * Resume a previously interrupted crawl from an existing archive file.
      *
@@ -408,7 +784,8 @@ export class CrawlerOrchestrator extends EventEmitter {
         }
         const { scraped, pending } = await archive.getCrawlingState();
         const resources = await archive.getResourceUrlList();
-        orchestrator.#crawler.resume(pending, scraped, resources);
+        const pagesScrapedOffset = await archive.getScrapedHtmlPageCount();
+        orchestrator.#crawler.resume(pending, scraped, resources, pagesScrapedOffset);
         if (initializedCallback) {
             await initializedCallback(orchestrator, config);
         }

package/lib/crawler.d.ts

@@ -17,6 +17,11 @@ export { default as Page } from './archive/page.js';
 export { default as ArchiveResource } from './archive/resource.js';
 export * from './archive/types.js';
 export { default as Archive } from './archive/archive.js';
+export { peekArchiveLockHolder } from './archive/peek-archive-lock.js';
+export type { ArchiveLockHolder } from './archive/peek-archive-lock.js';
+export type { FlatPageMetaColumns, JsonLdRow, JsonLdRowForInsert, TagRow, TagRowForInsert, JsonLdSummary, TagsSummary, TagInventoryEntry, PageDenormalizedColumns, } from './archive/meta/types.js';
+export { IncompatibleArchiveError } from './archive/meta/types.js';
+export { REQUIRED_FORMAT_VERSION } from './archive/meta/assert-compatible-version.js';
 export { DEFAULT_EXCLUDED_EXTERNAL_URLS, CrawlerOrchestrator, } from './crawler-orchestrator.js';
 export * from './types.js';
 export * from './crawler/types.js';

package/lib/crawler.js

@@ -17,6 +17,9 @@ export { default as Page } from './archive/page.js';
 export { default as ArchiveResource } from './archive/resource.js';
 export * from './archive/types.js';
 export { default as Archive } from './archive/archive.js';
+export { peekArchiveLockHolder } from './archive/peek-archive-lock.js';
+export { IncompatibleArchiveError } from './archive/meta/types.js';
+export { REQUIRED_FORMAT_VERSION } from './archive/meta/assert-compatible-version.js';
 // Core
 export { DEFAULT_EXCLUDED_EXTERNAL_URLS, CrawlerOrchestrator, } from './crawler-orchestrator.js';
 export * from './types.js';

package/lib/resource-row-to-lookup-result.d.ts

@@ -0,0 +1,13 @@
+import type { DB_Resource } from './archive/types.js';
+import type { ResourceLookupResult } from './crawler/types.js';
+/**
+ * Convert a raw `resources` table row into the minimal lookup result the
+ * crawler needs to reuse captured sub-resource data.
+ *
+ * Header parsing degrades to `null` on malformed JSON instead of throwing
+ * because a missing header set only loses fidelity — the reuse path stays
+ * valid.
+ * @param row - The raw database row.
+ * @returns The lookup result consumed by the crawler's resource-reuse hook.
+ */
+export declare function resourceRowToLookupResult(row: DB_Resource): ResourceLookupResult;

package/lib/resource-row-to-lookup-result.js

@@ -0,0 +1,20 @@
+import { parseResponseHeaders } from './utils/object/parse-response-headers.js';
+/**
+ * Convert a raw `resources` table row into the minimal lookup result the
+ * crawler needs to reuse captured sub-resource data.
+ *
+ * Header parsing degrades to `null` on malformed JSON instead of throwing
+ * because a missing header set only loses fidelity — the reuse path stays
+ * valid.
+ * @param row - The raw database row.
+ * @returns The lookup result consumed by the crawler's resource-reuse hook.
+ */
+export function resourceRowToLookupResult(row) {
+    return {
+        status: row.status,
+        statusText: row.statusText,
+        contentType: row.contentType,
+        contentLength: row.contentLength,
+        responseHeaders: parseResponseHeaders(row.responseHeaders),
+    };
+}

package/lib/types.d.ts

@@ -1,4 +1,4 @@
-import type { CrawlerError } from './utils/types/types.js';
+import type { CrawlerError, PageData } from './utils/types/types.js';
 /**
  * Event map for the `CrawlerOrchestrator` class.
  *
@@ -24,4 +24,14 @@ export interface CrawlEvent {
      * Emitted when an error occurs during crawling or archiving.
      */
     error: CrawlerError;
+    /**
+     * Emitted when a URL redirects to a destination already rendered during this
+     * crawl, so only the redirect edge is recorded and the destination is not
+     * re-rendered (#73). Mirrors the crawler's `redirect` event; useful for
+     * observing how much redirect-convergence work was skipped.
+     */
+    redirect: {
+        /** HEAD-resolved page data carrying the redirect chain (source → destination). */
+        result: PageData;
+    };
 }

package/lib/utils/object/parse-response-headers.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Parse a JSON-serialized HTTP response headers column from the archive
+ * database.
+ *
+ * The columns hold JSON produced by `JSON.stringify` at insert time; absent,
+ * malformed, or non-object JSON (the string `"null"`, arrays, primitives)
+ * degrades to `null` instead of throwing because a missing header set only
+ * loses fidelity — callers decide their own fallback (`?? {}` etc.).
+ * @param json - The raw column value.
+ * @returns The parsed header record, or `null` when absent or malformed.
+ */
+export declare function parseResponseHeaders(json: string | null): Record<string, string | string[] | undefined> | null;

package/lib/utils/object/parse-response-headers.js

@@ -0,0 +1,26 @@
+/**
+ * Parse a JSON-serialized HTTP response headers column from the archive
+ * database.
+ *
+ * The columns hold JSON produced by `JSON.stringify` at insert time; absent,
+ * malformed, or non-object JSON (the string `"null"`, arrays, primitives)
+ * degrades to `null` instead of throwing because a missing header set only
+ * loses fidelity — callers decide their own fallback (`?? {}` etc.).
+ * @param json - The raw column value.
+ * @returns The parsed header record, or `null` when absent or malformed.
+ */
+export function parseResponseHeaders(json) {
+    if (json == null) {
+        return null;
+    }
+    try {
+        const parsed = JSON.parse(json);
+        if (parsed == null || typeof parsed !== 'object' || Array.isArray(parsed)) {
+            return null;
+        }
+        return parsed;
+    }
+    catch {
+        return null;
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@nitpicker/crawler",
-	"version": "0.8.0",
+	"version": "0.11.0",
 	"description": "Web crawler engine with headless browser rendering and archive storage",
 	"author": "D-ZERO",
 	"license": "Apache-2.0",
@@ -27,8 +27,8 @@
 		"clean": "tsc --build --clean"
 	},
 	"dependencies": {
-		"@d-zero/beholder": "2.1.2",
-		"@d-zero/dealer": "1.7.0",
+		"@d-zero/beholder": "3.1.1",
+		"@d-zero/dealer": "1.9.0",
 		"@d-zero/fs": "0.2.2",
 		"@d-zero/shared": "0.20.1",
 		"ansi-colors": "4.1.3",
@@ -48,5 +48,5 @@
 		"@types/tar": "7.0.87",
 		"@types/unzipper": "0.10.11"
 	},
-	"gitHead": "8456d02c3052b2b3438f0d32e096ed7e916c5234"
+	"gitHead": "b11dbe3691746b3cb980476808d1f745bc6d0e98"
 }