@nitpicker/crawler 0.4.4 → 0.6.0

package/lib/archive/archive-accessor.d.ts

@@ -25,17 +25,17 @@ export declare class ArchiveAccessor extends EventEmitter<DatabaseEvent> {
      *                    When null, `setData` is not available.
      */
     constructor(tmpDir: string, db: Database, namespace?: string | null);
-    /**
-     * Retrieves the crawl configuration stored in the archive database.
-     * @returns The parsed {@link Config} object.
-     */
-    getConfig(): Promise<Config>;
     /**
      * Retrieves anchor (link) data for a specific page by its database ID.
      * @param pageId - The database ID of the page whose anchors to retrieve.
      * @returns An array of anchor records found on the page.
      */
     getAnchorsOnPage(pageId: number): Promise<any[]>;
+    /**
+     * Retrieves the crawl configuration stored in the archive database.
+     * @returns The parsed {@link Config} object.
+     */
+    getConfig(): Promise<Config>;
     /**
      * Reads custom data stored in the archive by name.
      * @param name - The base name of the data file (without extension).
@@ -58,6 +58,13 @@ export declare class ArchiveAccessor extends EventEmitter<DatabaseEvent> {
      * @returns The HTML content as a string, or null if the snapshot is not found or filePath is null.
      */
     getHtmlOfPage(filePath: string | null, openZipped?: boolean): Promise<string | null>;
+    /**
+     * Returns the underlying Knex query builder instance for direct SQL access.
+     * Enables advanced queries (GROUP BY, HAVING, JOINs) at the database layer
+     * for performance-critical operations on large datasets.
+     * @returns The Knex instance connected to the SQLite database.
+     */
+    getKnex(): import("knex").Knex<any, any[]>;
     /**
      * Retrieves all pages from the archive, optionally filtered by type.
      * Eagerly loads redirect relationships (`redirectFrom`) but does NOT load

package/lib/archive/archive-accessor.js

@@ -1,7 +1,12 @@
 import path from 'node:path';
+import { extractZip, unzip } from '@d-zero/fs/zip';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
 import { log } from './debug.js';
-import { exists, extractZip, outputJSON, outputText, readJSON, readText, unzip, } from './filesystem/index.js';
+import { exists } from './filesystem/exists.js';
+import { outputJSON } from './filesystem/output-json.js';
+import { outputText } from './filesystem/output-text.js';
+import { readJSON } from './filesystem/read-json.js';
+import { readText } from './filesystem/read-text.js';
 import Page from './page.js';
 import Resource from './resource.js';
 import { safePath } from './safe-path.js';
@@ -41,13 +46,6 @@ export class ArchiveAccessor extends EventEmitter {
             void this.emit('error', e);
         });
     }
-    /**
-     * Retrieves the crawl configuration stored in the archive database.
-     * @returns The parsed {@link Config} object.
-     */
-    async getConfig() {
-        return this.#db.getConfig();
-    }
     /**
      * Retrieves anchor (link) data for a specific page by its database ID.
      * @param pageId - The database ID of the page whose anchors to retrieve.
@@ -57,6 +55,13 @@ export class ArchiveAccessor extends EventEmitter {
         const refs = await this.#db.getAnchorsOnPage(pageId);
         return refs;
     }
+    /**
+     * Retrieves the crawl configuration stored in the archive database.
+     * @returns The parsed {@link Config} object.
+     */
+    async getConfig() {
+        return this.#db.getConfig();
+    }
     async getData(name, format = 'json') {
         const namespace = this.#namespace || '';
         const filePath = safePath(this.#tmpDir, namespace, `${name}.${format}`);
@@ -103,6 +108,15 @@ export class ArchiveAccessor extends EventEmitter {
         log('Succeeded: Extracts %s from zipped snapshots', name);
         return html;
     }
+    /**
+     * Returns the underlying Knex query builder instance for direct SQL access.
+     * Enables advanced queries (GROUP BY, HAVING, JOINs) at the database layer
+     * for performance-critical operations on large datasets.
+     * @returns The Knex instance connected to the SQLite database.
+     */
+    getKnex() {
+        return this.#db.getKnex();
+    }
     /**
      * Retrieves all pages from the archive, optionally filtered by type.
      * Eagerly loads redirect relationships (`redirectFrom`) but does NOT load

package/lib/archive/archive.d.ts

@@ -1,5 +1,5 @@
 import type { Config } from './types.js';
-import type { PageData, CrawlerError, Resource } from '../utils/index.js';
+import type { PageData, CrawlerError, Resource } from '../utils/types/types.js';
 import type { ParseURLOptions } from '@d-zero/shared/parse-url';
 import { ArchiveAccessor } from './archive-accessor.js';
 /**
@@ -34,11 +34,6 @@ export default class Archive extends ArchiveAccessor {
      * it is removed.
      */
     close(): Promise<void>;
-    /**
-     * Retrieves the crawl configuration stored in the archive database.
-     * @returns The configuration object.
-     */
-    getConfig(): Promise<Config>;
     /**
      * Retrieves the current crawling state, including lists of scraped and pending URLs.
      * @returns An object with `scraped` and `pending` URL arrays.
@@ -64,8 +59,11 @@ export default class Archive extends ArchiveAccessor {
     setExternalPage(pageInfo: PageData): Promise<void>;
     /**
      * Stores a crawled page's data in the archive database and optionally saves an HTML snapshot.
+     * If the snapshot file write fails, the HTML path in the database is cleared to prevent
+     * referencing a non-existent file, and the error is re-thrown.
      * @param pageInfo - The page data to store.
      * @returns The database ID of the stored page.
+     * @throws {Error} Re-throws any error from the snapshot file write after clearing the HTML path.
      */
     setPage(pageInfo: PageData): Promise<number>;
     /**
@@ -154,6 +152,11 @@ export default class Archive extends ArchiveAccessor {
      * @returns A formatted timestamp string.
      */
     static timestamp(): string;
+    /**
+     * Retrieves the crawl configuration stored in the archive database.
+     * @returns The configuration object.
+     */
+    getConfig(): Promise<Config>;
 }
 /**
  * Options for creating or opening an archive.

package/lib/archive/archive.js

@@ -1,8 +1,16 @@
 import path from 'node:path';
+import { zip } from '@d-zero/fs/zip';
 import { ArchiveAccessor } from './archive-accessor.js';
 import { Database } from './database.js';
 import { dbLog, log, saveLog } from './debug.js';
-import { appendText, exists, isDir, outputText, remove, rename, tar, untar, zip, } from './filesystem/index.js';
+import { appendText } from './filesystem/append-text.js';
+import { exists } from './filesystem/exists.js';
+import { isDir } from './filesystem/is-dir.js';
+import { outputText } from './filesystem/output-text.js';
+import { remove } from './filesystem/remove.js';
+import { rename } from './filesystem/rename.js';
+import { tar } from './filesystem/tar.js';
+import { untar } from './filesystem/untar.js';
 /**
  * Main archive class for creating, opening, resuming, and writing Nitpicker archive files (`.nitpicker`).
  *
@@ -74,13 +82,6 @@ export default class Archive extends ArchiveAccessor {
         await this.#db.destroy();
         log('Closing done');
     }
-    /**
-     * Retrieves the crawl configuration stored in the archive database.
-     * @returns The configuration object.
-     */
-    async getConfig() {
-        return this.#db.getConfig();
-    }
     /**
      * Retrieves the current crawling state, including lists of scraped and pending URLs.
      * @returns An object with `scraped` and `pending` URL arrays.
@@ -113,17 +114,30 @@ export default class Archive extends ArchiveAccessor {
     }
     /**
      * Stores a crawled page's data in the archive database and optionally saves an HTML snapshot.
+     * If the snapshot file write fails, the HTML path in the database is cleared to prevent
+     * referencing a non-existent file, and the error is re-thrown.
      * @param pageInfo - The page data to store.
      * @returns The database ID of the stored page.
+     * @throws {Error} Re-throws any error from the snapshot file write after clearing the HTML path.
      */
     async setPage(pageInfo) {
         dbLog('Set page: %s', pageInfo.url.href);
         const { html, pageId } = await this.#db.updatePage(pageInfo, this.#snapshotDir, pageInfo.isTarget);
-        const snapshotTask = [];
         if (html) {
-            snapshotTask.push(outputText(html, pageInfo.html));
+            try {
+                await outputText(html, pageInfo.html);
+            }
+            catch (error) {
+                dbLog('Snapshot write failed for page %d, clearing html path: %s', pageId, html);
+                try {
+                    await this.#db.clearHtmlPath(pageId);
+                }
+                catch (clearError) {
+                    dbLog('Failed to clear html path for page %d: %s', pageId, clearError);
+                }
+                throw error;
+            }
         }
-        await Promise.all(snapshotTask);
         return pageId;
     }
     /**
@@ -328,4 +342,11 @@ export default class Archive extends ArchiveAccessor {
         const archive = new Archive(filePath, tmpDir, db);
         return archive;
     }
+    /**
+     * Retrieves the crawl configuration stored in the archive database.
+     * @returns The configuration object.
+     */
+    async getConfig() {
+        return this.#db.getConfig();
+    }
 }

package/lib/archive/database.d.ts

@@ -1,5 +1,6 @@
 import type { Config, DB_Anchor, DB_Page, DB_Redirect, DB_Referrer, DB_Resource, DatabaseEvent, PageFilter } from './types.js';
-import type { PageData, Resource } from '../utils/index.js';
+import type { PageData, Resource } from '../utils/types/types.js';
+import type { Knex } from 'knex';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
 /**
  * Low-level database abstraction layer for the archive's SQLite database.
@@ -17,8 +18,9 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
     private constructor();
     /**
      * Adds the `order` column to the `pages` table for URL sort ordering.
+     * If the column already exists, this method does nothing.
      * @deprecated Since v0.1.x. The column is now created during table initialization.
-     * @returns The result of the schema alteration.
+     * @returns The result of the schema alteration, or void if the column already exists.
      */
     addOrderField(): Promise<void>;
     /**
@@ -27,6 +29,15 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * This ensures the database is fully self-contained in `db.sqlite` before archiving.
      */
     checkpoint(): Promise<void>;
+    /**
+     * Clears the HTML snapshot path for a page.
+     * Used to roll back the snapshot reference when the snapshot file write fails.
+     * @param pageId - The database ID of the page whose HTML path should be cleared.
+     */
+    clearHtmlPath(pageId: number): Promise<void>;
+    /**
+     * Destroys the database connection, releasing all pooled resources.
+     */
     destroy(): Promise<void>;
     /**
      * Retrieves all anchors (outgoing links) on a specific page.
@@ -62,6 +73,13 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @returns The relative file path to the HTML snapshot, or null if not saved.
      */
     getHtmlPathOnPage(pageId: number): Promise<any>;
+    /**
+     * Returns the underlying Knex query builder instance for direct SQL access.
+     * This enables advanced queries (GROUP BY, HAVING, JOINs) at the database
+     * layer for performance with large datasets.
+     * @returns The Knex instance connected to the SQLite database.
+     */
+    getKnex(): Knex;
     /**
      * Retrieves the crawl session name from the `info` table.
      * @returns The name string.

package/lib/archive/database.js

@@ -38,9 +38,10 @@ import { retry } from '@d-zero/shared/retry';
 import { pathComparator } from '@d-zero/shared/sort/path';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
 import knex from 'knex';
-import { ErrorEmitter, eachSplitted } from '../utils/index.js';
+import { eachSplitted } from '../utils/array/each-splitted.js';
+import { ErrorEmitter } from '../utils/error/error-emitter.js';
 import { dbLog } from './debug.js';
-import { mkdir } from './filesystem/index.js';
+import { mkdir } from './filesystem/mkdir.js';
 const retrySetting = {
     interval: 300,
     retries: 3,
@@ -59,6 +60,7 @@ const retrySetting = {
 let Database = (() => {
     let _classSuper = EventEmitter;
     let _instanceExtraInitializers = [];
+    let _clearHtmlPath_decorators;
     let _getAnchorsOnPage_decorators;
     let _getBaseUrl_decorators;
     let _getConfig_decorators;
@@ -81,6 +83,7 @@ let Database = (() => {
     return class Database extends _classSuper {
         static {
             const _metadata = typeof Symbol === "function" && Symbol.metadata ? Object.create(_classSuper[Symbol.metadata] ?? null) : void 0;
+            _clearHtmlPath_decorators = [ErrorEmitter(), retry(retrySetting)];
             _getAnchorsOnPage_decorators = [ErrorEmitter(), retry(retrySetting)];
             _getBaseUrl_decorators = [ErrorEmitter(), retry(retrySetting)];
             _getConfig_decorators = [ErrorEmitter(), retry(retrySetting)];
@@ -100,6 +103,7 @@ let Database = (() => {
             _setConfig_decorators = [ErrorEmitter(), retry(retrySetting)];
             _setSkippedPage_decorators = [ErrorEmitter(), retry(retrySetting)];
             _updatePage_decorators = [ErrorEmitter(), retry(retrySetting)];
+            __esDecorate(this, null, _clearHtmlPath_decorators, { kind: "method", name: "clearHtmlPath", static: false, private: false, access: { has: obj => "clearHtmlPath" in obj, get: obj => obj.clearHtmlPath }, metadata: _metadata }, null, _instanceExtraInitializers);
             __esDecorate(this, null, _getAnchorsOnPage_decorators, { kind: "method", name: "getAnchorsOnPage", static: false, private: false, access: { has: obj => "getAnchorsOnPage" in obj, get: obj => obj.getAnchorsOnPage }, metadata: _metadata }, null, _instanceExtraInitializers);
             __esDecorate(this, null, _getBaseUrl_decorators, { kind: "method", name: "getBaseUrl", static: false, private: false, access: { has: obj => "getBaseUrl" in obj, get: obj => obj.getBaseUrl }, metadata: _metadata }, null, _instanceExtraInitializers);
             __esDecorate(this, null, _getConfig_decorators, { kind: "method", name: "getConfig", static: false, private: false, access: { has: obj => "getConfig" in obj, get: obj => obj.getConfig }, metadata: _metadata }, null, _instanceExtraInitializers);
@@ -150,10 +154,15 @@ let Database = (() => {
         }
         /**
          * Adds the `order` column to the `pages` table for URL sort ordering.
+         * If the column already exists, this method does nothing.
          * @deprecated Since v0.1.x. The column is now created during table initialization.
-         * @returns The result of the schema alteration.
+         * @returns The result of the schema alteration, or void if the column already exists.
          */
         async addOrderField() {
+            const hasColumn = await this.#instance.schema.hasColumn('pages', 'order');
+            if (hasColumn) {
+                return;
+            }
             return await this.#instance.schema.table('pages', (t) => {
                 t.integer('order').unsigned().nullable().defaultTo(null);
             });
@@ -166,6 +175,17 @@ let Database = (() => {
         async checkpoint() {
             await this.#instance.raw('PRAGMA wal_checkpoint(TRUNCATE)');
         }
+        /**
+         * Clears the HTML snapshot path for a page.
+         * Used to roll back the snapshot reference when the snapshot file write fails.
+         * @param pageId - The database ID of the page whose HTML path should be cleared.
+         */
+        async clearHtmlPath(pageId) {
+            await this.#instance('pages').where('id', pageId).update({ html: null });
+        }
+        /**
+         * Destroys the database connection, releasing all pooled resources.
+         */
         async destroy() {
             await this.#instance.destroy();
         }
@@ -255,6 +275,15 @@ let Database = (() => {
                 return html || null;
             });
         }
+        /**
+         * Returns the underlying Knex query builder instance for direct SQL access.
+         * This enables advanced queries (GROUP BY, HAVING, JOINs) at the database
+         * layer for performance with large datasets.
+         * @returns The Knex instance connected to the SQLite database.
+         */
+        getKnex() {
+            return this.#instance;
+        }
         /**
          * Retrieves the crawl session name from the `info` table.
          * @returns The name string.
@@ -374,7 +403,7 @@ let Database = (() => {
          * @returns An object containing `pages`, `redirects`, `anchors`, and `referrers` arrays.
          */
         async getPagesWithRels(offset, limit) {
-            await this.addOrderField().catch((error) => error);
+            await this.addOrderField();
             await this.setUrlOrder();
             dbLog('Get Pages');
             const pages = await this.#instance
@@ -952,6 +981,9 @@ function redirectTable(includeNull = true) {
 // ----- ----- ----- ----- -----
 /**
  * Safely parses a JSON string, returning a fallback value if parsing fails or the input is not a string.
+ * Logs a warning via {@link dbLog} when invalid JSON is detected, including a truncated preview
+ * of the data and the parse error message.
+ * @template T The expected type of the parsed JSON value and the fallback.
  * @param data - The data to parse. Only string values are parsed; other types return the fallback.
  * @param fallback - The value to return if parsing fails or the result is falsy.
  * @returns The parsed JSON value, or the fallback.
@@ -966,8 +998,8 @@ function getJSON(data, fallback) {
             return fallback;
         }
     }
-    catch {
-        // void
+    catch (error) {
+        dbLog('Warning: Invalid JSON detected in database field. Using fallback value. Data: %s, Error: %s', String(data).slice(0, 200), error instanceof Error ? error.message : String(error));
     }
     return fallback;
 }

package/lib/archive/filesystem/rename.d.ts

@@ -4,8 +4,13 @@
  * If `override` is `true`, the destination is unconditionally removed
  * before renaming. This avoids a TOCTOU race condition between
  * checking existence and performing the removal.
+ *
+ * When `fs.rename` fails with `EPERM` (common on Windows due to file locks
+ * from antivirus or indexer processes) or `EXDEV` (cross-device move),
+ * falls back to a copy-then-remove strategy.
  * @param oldPath - The current path of the file or directory.
  * @param newPath - The new path for the file or directory.
  * @param override - Whether to overwrite the destination if it already exists. Defaults to `false`.
+ * @returns A promise that resolves when the rename (or fallback copy-then-remove) is complete.
  */
 export declare function rename(oldPath: string, newPath: string, override?: boolean): Promise<void>;

package/lib/archive/filesystem/rename.js

@@ -1,18 +1,48 @@
 import { promises as fs } from 'node:fs';
 import { remove } from './remove.js';
+/**
+ * Checks whether a value is a {@link NodeJS.ErrnoException}.
+ * @param error - The value to check.
+ * @returns `true` if the value has a `code` property of type `string`.
+ */
+function isNodeError(error) {
+    return (error instanceof Error && typeof error.code === 'string');
+}
 /**
  * Renames (moves) a file or directory from one path to another.
  *
  * If `override` is `true`, the destination is unconditionally removed
  * before renaming. This avoids a TOCTOU race condition between
  * checking existence and performing the removal.
+ *
+ * When `fs.rename` fails with `EPERM` (common on Windows due to file locks
+ * from antivirus or indexer processes) or `EXDEV` (cross-device move),
+ * falls back to a copy-then-remove strategy.
  * @param oldPath - The current path of the file or directory.
  * @param newPath - The new path for the file or directory.
  * @param override - Whether to overwrite the destination if it already exists. Defaults to `false`.
+ * @returns A promise that resolves when the rename (or fallback copy-then-remove) is complete.
  */
 export async function rename(oldPath, newPath, override = false) {
     if (override) {
         await remove(newPath).catch(() => { });
     }
-    await fs.rename(oldPath, newPath);
+    try {
+        await fs.rename(oldPath, newPath);
+    }
+    catch (error) {
+        if (isNodeError(error) && (error.code === 'EPERM' || error.code === 'EXDEV')) {
+            try {
+                await fs.cp(oldPath, newPath, { recursive: true });
+            }
+            catch (cpError) {
+                await remove(newPath).catch(() => { });
+                throw cpError;
+            }
+            await remove(oldPath);
+        }
+        else {
+            throw error;
+        }
+    }
 }

package/lib/archive/types.d.ts

@@ -183,6 +183,35 @@ export interface DB_Referrer {
     /** The text content of the referring anchor element, or null if empty. */
     textContent: string | null;
 }
+/**
+ * Raw database row representing an image element found on a page in the `images` table.
+ */
+export interface DB_Image {
+    /** Auto-incremented primary key. */
+    id: number;
+    /** Foreign key to the page that contains this image. */
+    pageId: number;
+    /** The `src` attribute value of the image element. */
+    src: string | null;
+    /** The actual loaded source URL of the image (after srcset/picture resolution). */
+    currentSrc: string | null;
+    /** The `alt` attribute value, or null if not present. */
+    alt: string | null;
+    /** The rendered width of the image in CSS pixels. */
+    width: number;
+    /** The rendered height of the image in CSS pixels. */
+    height: number;
+    /** The intrinsic width of the image in pixels. */
+    naturalWidth: number;
+    /** The intrinsic height of the image in pixels. */
+    naturalHeight: number;
+    /** Whether the image uses lazy loading. */
+    isLazy: number | null;
+    /** The viewport width at the time of capture. */
+    viewportWidth: number;
+    /** The raw HTML source code of the image element. */
+    sourceCode: string | null;
+}
 /**
  * Raw database row representing a sub-resource (CSS, JS, image, etc.) in the `resources` table.
  */

package/lib/crawler/crawler.d.ts

@@ -15,6 +15,13 @@ export type { CrawlerOptions } from './types.js';
  */
 export default class Crawler extends EventEmitter<CrawlerEventTypes> {
     #private;
+    /**
+     * The AbortSignal associated with this crawler's AbortController.
+     *
+     * Passed to `deal()` so that it stops launching new workers after abort.
+     * Also available to the orchestrator for forwarding to other subsystems.
+     */
+    get signal(): AbortSignal;
     /**
      * Create a new Crawler instance.
      * @param options - Configuration options for crawling behavior. All fields have
@@ -24,8 +31,10 @@ export default class Crawler extends EventEmitter<CrawlerEventTypes> {
     /**
      * Abort the current crawl operation.
      *
-     * Sets the aborted flag and immediately emits a `crawlEnd` event.
-     * In-progress scrape tasks will check the flag and exit early.
+     * Signals the AbortController so that the dealer stops launching new
+     * workers. Currently running workers will finish, after which `deal()`
+     * resolves and `crawlEnd` is emitted by the normal completion path in
+     * {@link #runDeal}.
      */
     abort(): void;
     /**

package/lib/crawler/crawler.js

@@ -11,6 +11,7 @@ import pkg from '../../package.json' with { type: 'json' };
 import { crawlerLog } from '../debug.js';
 import { detectPaginationPattern } from './detect-pagination-pattern.js';
 import { fetchDestination } from './fetch-destination.js';
+import { formatCrawlProgress } from './format-crawl-progress.js';
 import { generatePredictedUrls } from './generate-predicted-urls.js';
 import { handleIgnoreAndSkip } from './handle-ignore-and-skip.js';
 import { handleResourceResponse } from './handle-resource-response.js';
@@ -36,8 +37,8 @@ import { shouldSkipUrl } from './should-skip-url.js';
  * configurable parallelism up to {@link Crawler.MAX_PROCESS_LENGTH}.
  */
 class Crawler extends EventEmitter {
-    /** Flag set by `abort()` to signal in-progress tasks to exit early. */
-    #aborted = false;
+    /** Controller used to cancel the deal-based crawl via its AbortSignal. */
+    #abortController = new AbortController();
     /** Tracks discovered URLs, their scrape status, and deduplication. */
     #linkList = new LinkList();
     /** Merged crawler configuration (user overrides + defaults). */
@@ -52,6 +53,15 @@ class Crawler extends EventEmitter {
     #robotsChecker;
     /** Maps hostnames to their scope URLs. Defines the crawl boundary for internal/external classification. */
     #scope = new Map();
+    /**
+     * The AbortSignal associated with this crawler's AbortController.
+     *
+     * Passed to `deal()` so that it stops launching new workers after abort.
+     * Also available to the orchestrator for forwarding to other subsystems.
+     */
+    get signal() {
+        return this.#abortController.signal;
+    }
     /**
      * Create a new Crawler instance.
      * @param options - Configuration options for crawling behavior. All fields have
@@ -90,12 +100,13 @@ class Crawler extends EventEmitter {
     /**
      * Abort the current crawl operation.
      *
-     * Sets the aborted flag and immediately emits a `crawlEnd` event.
-     * In-progress scrape tasks will check the flag and exit early.
+     * Signals the AbortController so that the dealer stops launching new
+     * workers. Currently running workers will finish, after which `deal()`
+     * resolves and `crawlEnd` is emitted by the normal completion path in
+     * {@link #runDeal}.
      */
     abort() {
-        this.#aborted = true;
-        void this.emit('crawlEnd', {});
+        this.#abortController.abort();
     }
     /**
      * Retrieve the list of Chromium process IDs that are still running.
@@ -147,12 +158,7 @@ class Crawler extends EventEmitter {
         }
         void this.#runDeal(initialUrls, resumeOffset).catch((error) => {
             crawlerLog('runDeal error: %O', error);
-            void this.emit('error', {
-                pid: process.pid,
-                isMainProcess: true,
-                url: url.href,
-                error: error instanceof Error ? error : new Error(String(error)),
-            });
+            this.#emitDealErrors(error, url.href);
             void this.emit('crawlEnd', {});
         });
     }
@@ -183,14 +189,30 @@ class Crawler extends EventEmitter {
         this.#options.fromList = true;
         void this.#runDeal(pageList).catch((error) => {
             crawlerLog('runDeal error: %O', error);
+            this.#emitDealErrors(error, pageList[0].href);
+            void this.emit('crawlEnd', {});
+        });
+    }
+    /**
+     * Emits error events for a deal-level failure.
+     *
+     * When the dealer rejects with an `AggregateError` (e.g. multiple worker
+     * failures), each inner error is emitted as a separate `error` event.
+     * For any other error type, a single `error` event is emitted.
+     * @param error - The error thrown by `#runDeal`.
+     * @param fallbackUrl - URL string used as the error context (typically the root URL).
+     */
+    #emitDealErrors(error, fallbackUrl) {
+        const errors = error instanceof AggregateError ? error.errors : [error];
+        for (const e of errors) {
             void this.emit('error', {
                 pid: process.pid,
                 isMainProcess: true,
-                url: pageList[0].href,
-                error: error instanceof Error ? error : new Error(String(error)),
+                url: fallbackUrl,
+                isExternal: false,
+                error: e instanceof Error ? e : new Error(String(e)),
             });
-            void this.emit('crawlEnd', {});
-        });
+        }
     }
     /**
      * Processes captured sub-resources from a page scrape, deduplicates them,
@@ -295,6 +317,7 @@ class Crawler extends EventEmitter {
                     shutdown: result.error.shutdown,
                     pid: undefined,
                 }, this.#linkList, this.#scope, this.#options);
+                const isExternal = isExternalUrl(url, this.#scope);
                 if (pageResult) {
                     if (pageResult.isExternal) {
                         void this.emit('externalPage', { result: pageResult });
@@ -307,6 +330,7 @@ class Crawler extends EventEmitter {
                     pid: process.pid,
                     isMainProcess: true,
                     url: url.href,
+                    isExternal,
                     error,
                 });
                 break;
@@ -423,8 +447,6 @@ class Crawler extends EventEmitter {
             this.#linkList.add(url);
             this.#linkList.progress(url);
             return async () => {
-                if (this.#aborted)
-                    return;
                 const log = createTimedUpdate(update, this.#options.verbose);
                 try {
                     const robotsAllowed = await this.#robotsChecker.isAllowed(url);
@@ -476,6 +498,24 @@ class Crawler extends EventEmitter {
                     this.#handleResources(result.resources);
                     log(formatResultSummary(result));
                 }
+                catch (error) {
+                    crawlerLog('Worker error for %s: %O', url.href, error);
+                    log(c.red('Error'));
+                    const workerError = error instanceof Error ? error : new Error(String(error));
+                    handleScrapeError({
+                        url,
+                        error: workerError,
+                        shutdown: false,
+                        pid: process.pid,
+                    }, this.#linkList, this.#scope, this.#options);
+                    void this.emit('error', {
+                        pid: process.pid,
+                        isMainProcess: true,
+                        url: url.href,
+                        isExternal,
+                        error: workerError,
+                    });
+                }
                 finally {
                     if (isExternal) {
                         externalDoneUrls.add(protocolAgnosticKey(url.withoutHashAndAuth));
@@ -486,15 +526,16 @@ class Crawler extends EventEmitter {
             limit: concurrency,
             interval: this.#options.interval,
             verbose: this.#options.verbose || !process.stdout.isTTY,
+            signal: this.#abortController.signal,
             header: (_progress, done, total, limit) => {
-                const allDone = done + resumeOffset;
-                const allTotal = total + resumeOffset;
-                const extTotal = externalUrls.size;
-                const extDone = externalDoneUrls.size;
-                const pct = allTotal > 0 ? Math.round((allDone / allTotal) * 100) : 0;
-                return (c.bold(`Crawling: ${allDone - extDone}/${allTotal - extTotal}`) +
-                    c.dim(`(${extDone}/${extTotal})`) +
-                    c.bold(` (${pct}%) [${limit} parallel]`));
+                return formatCrawlProgress({
+                    done,
+                    total,
+                    resumeOffset,
+                    externalTotal: externalUrls.size,
+                    externalDone: externalDoneUrls.size,
+                    limit,
+                });
             },
             onPush: (url) => {
                 const key = protocolAgnosticKey(url.withoutHashAndAuth);

package/lib/crawler/format-crawl-progress.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Parameters for formatting crawl progress display.
+ */
+interface FormatCrawlProgressParams {
+    /** Number of URLs completed by the deal queue */
+    readonly done: number;
+    /** Total number of URLs in the deal queue (including completed) */
+    readonly total: number;
+    /** Offset from a previous resumed session */
+    readonly resumeOffset: number;
+    /** Number of external URLs discovered */
+    readonly externalTotal: number;
+    /** Number of external URLs completed */
+    readonly externalDone: number;
+    /** Number of parallel workers */
+    readonly limit: number;
+}
+/**
+ * Formats the crawl progress header for the deal() progress display.
+ *
+ * Shows "done / found (remaining)" format instead of "done/total"
+ * to make it clearer that the total is expected to grow during crawling.
+ * @param params - The crawl progress parameters.
+ * @param params.done - Number of URLs completed by the deal queue.
+ * @param params.total - Total number of URLs in the deal queue (including completed).
+ * @param params.resumeOffset - Offset from a previous resumed session.
+ * @param params.externalTotal - Number of external URLs discovered.
+ * @param params.externalDone - Number of external URLs completed.
+ * @param params.limit - Number of parallel workers.
+ * @returns The formatted progress string with ANSI color codes.
+ */
+export declare function formatCrawlProgress({ done, total, resumeOffset, externalTotal, externalDone, limit, }: FormatCrawlProgressParams): string;
+export {};

package/lib/crawler/format-crawl-progress.js

@@ -0,0 +1,29 @@
+import c from 'ansi-colors';
+/**
+ * Formats the crawl progress header for the deal() progress display.
+ *
+ * Shows "done / found (remaining)" format instead of "done/total"
+ * to make it clearer that the total is expected to grow during crawling.
+ * @param params - The crawl progress parameters.
+ * @param params.done - Number of URLs completed by the deal queue.
+ * @param params.total - Total number of URLs in the deal queue (including completed).
+ * @param params.resumeOffset - Offset from a previous resumed session.
+ * @param params.externalTotal - Number of external URLs discovered.
+ * @param params.externalDone - Number of external URLs completed.
+ * @param params.limit - Number of parallel workers.
+ * @returns The formatted progress string with ANSI color codes.
+ */
+export function formatCrawlProgress({ done, total, resumeOffset, externalTotal, externalDone, limit, }) {
+    const allDone = done + resumeOffset;
+    const allTotal = total + resumeOffset;
+    const internalDone = allDone - externalDone;
+    const internalTotal = allTotal - externalTotal;
+    const internalRemaining = internalTotal - internalDone;
+    const externalRemaining = externalTotal - externalDone;
+    const totalRemaining = internalRemaining + externalRemaining;
+    const pct = allTotal > 0 ? Math.round((allDone / allTotal) * 100) : 0;
+    return (c.bold(`Crawling: ${internalDone} done / ${internalTotal} found`) +
+        c.dim(` (+${externalDone}/${externalTotal} ext)`) +
+        c.bold(` (${pct}%) [${totalRemaining} remaining]`) +
+        c.dim(` [${limit} parallel]`));
+}

package/lib/crawler/handle-ignore-and-skip.d.ts

@@ -1,6 +1,6 @@
 import type LinkList from './link-list.js';
 import type { CrawlerOptions } from './types.js';
-import type { Link } from '../utils/index.js';
+import type { Link } from '../utils/types/types.js';
 import type { ExURL } from '@d-zero/shared/parse-url';
 /**
  * Handle a URL that was ignored or skipped during scraping.

package/lib/crawler/handle-resource-response.d.ts

@@ -1,4 +1,4 @@
-import type { Resource } from '../utils/index.js';
+import type { Resource } from '../utils/types/types.js';
 /**
  * Track a network resource response and determine if it is newly discovered.
  *

package/lib/crawler/handle-scrape-end.d.ts

@@ -1,6 +1,6 @@
 import type LinkList from './link-list.js';
 import type { CrawlerOptions } from './types.js';
-import type { Link, PageData } from '../utils/index.js';
+import type { Link, PageData } from '../utils/types/types.js';
 import type { ExURL } from '@d-zero/shared/parse-url';
 /**
  * Process the result of a successful page scrape.

package/lib/crawler/handle-scrape-error.d.ts

@@ -1,6 +1,6 @@
 import type LinkList from './link-list.js';
 import type { CrawlerOptions } from './types.js';
-import type { Link, PageData } from '../utils/index.js';
+import type { Link, PageData } from '../utils/types/types.js';
 import type { ExURL } from '@d-zero/shared/parse-url';
 /**
  * Handle an error that occurred during page scraping.

package/lib/crawler/link-list.d.ts

@@ -1,4 +1,4 @@
-import type { Link, PageData } from '../utils/index.js';
+import type { Link, PageData } from '../utils/types/types.js';
 import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
 /**
  * Manages the queue of URLs discovered during crawling.

package/lib/crawler/link-to-page-data.d.ts

@@ -1,4 +1,4 @@
-import type { Link, PageData } from '../utils/index.js';
+import type { Link, PageData } from '../utils/types/types.js';
 /**
  * Convert a {@link Link} object into a {@link PageData} structure.
  *

package/lib/crawler/types.d.ts

@@ -1,4 +1,4 @@
-import type { PageData, CrawlerError, Resource } from '../utils/index.js';
+import type { PageData, CrawlerError, Resource } from '../utils/types/types.js';
 import type { ChangePhaseEvent } from '@d-zero/beholder';
 import type { ParseURLOptions } from '@d-zero/shared/parse-url';
 /**

package/lib/crawler-orchestrator.d.ts

@@ -69,11 +69,11 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
     get archive(): Archive;
     private constructor();
     /**
-     * Abort the current crawl and archive operations.
+     * Abort the current crawl operation.
      *
-     * Delegates to the archive's abort method, which stops all in-progress
-     * database writes and cleans up temporary resources.
-     * @returns The result of the archive abort operation.
+     * Delegates to the crawler's AbortController so that the dealer stops
+     * launching new workers. Currently running workers will finish, after
+     * which `deal()` resolves and `crawlEnd` is emitted normally.
      */
     abort(): void;
     /**

package/lib/crawler-orchestrator.js

@@ -1,11 +1,15 @@
+import path from 'node:path';
 import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
 import { sortUrl } from '@d-zero/shared/sort-url';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
 import pkg from '../package.json' with { type: 'json' };
 import Archive from './archive/archive.js';
-import { clearDestinationCache, Crawler } from './crawler/index.js';
+import { clearDestinationCache } from './crawler/clear-destination-cache.js';
+import Crawler from './crawler/crawler.js';
 import { crawlerLog, log } from './debug.js';
-import { cleanObject } from './utils/index.js';
+import { resolveOutputPath } from './resolve-output-path.js';
+import { cleanObject } from './utils/object/clean-object.js';
+import { WriteQueue } from './write-queue.js';
 /**
  * Default list of external URL prefixes excluded from crawling.
  * Includes social media sharing endpoints that are commonly linked
@@ -64,6 +68,7 @@ export class CrawlerOrchestrator extends EventEmitter {
                 pid: process.pid,
                 isMainProcess: true,
                 url: null,
+                isExternal: false,
                 error: e instanceof Error ? e : new Error(String(e)),
             });
         });
@@ -91,14 +96,14 @@ export class CrawlerOrchestrator extends EventEmitter {
         });
     }
     /**
-     * Abort the current crawl and archive operations.
+     * Abort the current crawl operation.
      *
-     * Delegates to the archive's abort method, which stops all in-progress
-     * database writes and cleans up temporary resources.
-     * @returns The result of the archive abort operation.
+     * Delegates to the crawler's AbortController so that the dealer stops
+     * launching new workers. Currently running workers will finish, after
+     * which `deal()` resolves and `crawlEnd` is emitted normally.
      */
     abort() {
-        return this.#archive.abort();
+        this.#crawler.abort();
     }
     /**
      * Execute the crawl for the given list of URLs.
@@ -115,31 +120,43 @@ export class CrawlerOrchestrator extends EventEmitter {
         if (!root) {
             throw new Error('URL is empty');
         }
+        const writeQueue = new WriteQueue();
         return new Promise((resolve, reject) => {
             this.#crawler.on('error', (error) => {
                 crawlerLog('On error: %O', error);
-                void this.#archive.addError(error);
+                void writeQueue.enqueue(() => this.#archive.addError(error));
                 void this.emit('error', error);
             });
-            this.#crawler.on('page', async ({ result }) => {
-                await this.#archive.setPage(result).catch((error) => reject(error));
+            this.#crawler.on('page', ({ result }) => {
+                writeQueue
+                    .enqueue(() => this.#archive.setPage(result))
+                    .catch((error) => reject(error));
             });
             this.#crawler.on('externalPage', ({ result }) => {
-                this.#archive.setExternalPage(result).catch((error) => reject(error));
+                writeQueue
+                    .enqueue(() => this.#archive.setExternalPage(result))
+                    .catch((error) => reject(error));
             });
             this.#crawler.on('skip', ({ url, reason, isExternal }) => {
-                this.#archive
-                    .setSkippedPage(url, reason, isExternal)
+                writeQueue
+                    .enqueue(() => this.#archive.setSkippedPage(url, reason, isExternal))
                     .catch((error) => reject(error));
             });
             this.#crawler.on('response', ({ resource }) => {
-                this.#archive.setResources(resource).catch((error) => reject(error));
+                writeQueue
+                    .enqueue(() => this.#archive.setResources(resource))
+                    .catch((error) => reject(error));
             });
             this.#crawler.on('responseReferrers', (resource) => {
-                this.#archive.setResourcesReferrers(resource).catch((error) => reject(error));
+                writeQueue
+                    .enqueue(() => this.#archive.setResourcesReferrers(resource))
+                    .catch((error) => reject(error));
             });
             this.#crawler.on('crawlEnd', () => {
-                resolve();
+                writeQueue
+                    .drain()
+                    .then(() => resolve())
+                    .catch((error) => reject(error));
             });
             if (this.#fromList) {
                 this.#crawler.startMultiple(list);
@@ -211,9 +228,11 @@ export class CrawlerOrchestrator extends EventEmitter {
         if (!urlParsed) {
             throw new Error('URL is empty');
         }
-        const fileName = `${urlParsed.hostname}-${Archive.timestamp()}`;
         const cwd = options?.cwd ?? process.cwd();
-        const filePath = Archive.joinPath(cwd, `${fileName}.${Archive.FILE_EXTENSION}`);
+        const filePath = options?.filePath
+            ? resolveOutputPath(options.filePath, cwd)
+            : Archive.joinPath(cwd, `${urlParsed.hostname}-${Archive.timestamp()}.${Archive.FILE_EXTENSION}`);
+        const fileName = path.basename(filePath, `.${Archive.FILE_EXTENSION}`) || path.basename(filePath);
         const disableQueries = options?.disableQueries || false;
         const defaultUserAgent = `Nitpicker/${pkg.version}`;
         const archive = await Archive.create({ filePath, cwd, disableQueries });
@@ -227,14 +246,11 @@ export class CrawlerOrchestrator extends EventEmitter {
             interval: options?.interval || 0,
             parallels: options?.parallels || 0,
             scope: options?.scope ?? [],
-            // @ts-expect-error TODO: Fix CLI arguments
-            excludes: normalizeToArray(options?.exclude),
-            // @ts-expect-error TODO: Fix CLI arguments
-            excludeKeywords: normalizeToArray(options?.excludeKeyword),
+            excludes: normalizeToArray(options?.excludes),
+            excludeKeywords: normalizeToArray(options?.excludeKeywords),
             excludeUrls: [
                 ...DEFAULT_EXCLUDED_EXTERNAL_URLS,
-                // @ts-expect-error TODO: Fix CLI arguments
-                ...normalizeToArray(options?.excludeUrl),
+                ...normalizeToArray(options?.excludeUrls),
             ],
             maxExcludedDepth: options?.maxExcludedDepth || 10,
             retry: options?.retry ?? 3,

package/lib/crawler.d.ts

@@ -0,0 +1,22 @@
+/**
+ * @module @nitpicker/crawler
+ *
+ * Core module of Nitpicker that provides the main crawling engine,
+ * utility functions, type definitions, and archive storage layer.
+ */
+export * from './utils/types/types.js';
+export { eachSplitted } from './utils/array/each-splitted.js';
+export { DOMEvaluationError } from './utils/error/dom-evaluation-error.js';
+export { ErrorEmitter } from './utils/error/error-emitter.js';
+export type { ErrorEvent } from './utils/error/error-emitter.js';
+export * from './utils/object/clean-object.js';
+export { globalLog as log } from './utils/debug.js';
+export { ArchiveAccessor } from './archive/archive-accessor.js';
+export type { Redirect, Referrer, Anchor, StaticPageData } from './archive/page.js';
+export { default as Page } from './archive/page.js';
+export { default as ArchiveResource } from './archive/resource.js';
+export * from './archive/types.js';
+export { default as Archive } from './archive/archive.js';
+export { DEFAULT_EXCLUDED_EXTERNAL_URLS, CrawlerOrchestrator, } from './crawler-orchestrator.js';
+export * from './types.js';
+export * from './crawler/types.js';

package/lib/crawler.js

@@ -0,0 +1,23 @@
+/**
+ * @module @nitpicker/crawler
+ *
+ * Core module of Nitpicker that provides the main crawling engine,
+ * utility functions, type definitions, and archive storage layer.
+ */
+// Types + Utils (旧 @nitpicker/types + utils)
+export * from './utils/types/types.js';
+export { eachSplitted } from './utils/array/each-splitted.js';
+export { DOMEvaluationError } from './utils/error/dom-evaluation-error.js';
+export { ErrorEmitter } from './utils/error/error-emitter.js';
+export * from './utils/object/clean-object.js';
+export { globalLog as log } from './utils/debug.js';
+// Archive
+export { ArchiveAccessor } from './archive/archive-accessor.js';
+export { default as Page } from './archive/page.js';
+export { default as ArchiveResource } from './archive/resource.js';
+export * from './archive/types.js';
+export { default as Archive } from './archive/archive.js';
+// Core
+export { DEFAULT_EXCLUDED_EXTERNAL_URLS, CrawlerOrchestrator, } from './crawler-orchestrator.js';
+export * from './types.js';
+export * from './crawler/types.js';

package/lib/resolve-output-path.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Resolves and validates an output file path for the `.nitpicker` archive.
+ *
+ * Converts relative paths to absolute using the given working directory,
+ * appends the `.nitpicker` extension if missing, and verifies that the
+ * parent directory exists.
+ * @param outputPath - The user-specified output file path (relative or absolute).
+ * @param cwd - The working directory used to resolve relative paths.
+ * @returns The resolved absolute file path with the `.nitpicker` extension.
+ * @throws {Error} If the parent directory of the resolved path does not exist.
+ */
+export declare function resolveOutputPath(outputPath: string, cwd: string): string;

package/lib/resolve-output-path.js

@@ -0,0 +1,26 @@
+import { existsSync } from 'node:fs';
+import path from 'node:path';
+import Archive from './archive/archive.js';
+/**
+ * Resolves and validates an output file path for the `.nitpicker` archive.
+ *
+ * Converts relative paths to absolute using the given working directory,
+ * appends the `.nitpicker` extension if missing, and verifies that the
+ * parent directory exists.
+ * @param outputPath - The user-specified output file path (relative or absolute).
+ * @param cwd - The working directory used to resolve relative paths.
+ * @returns The resolved absolute file path with the `.nitpicker` extension.
+ * @throws {Error} If the parent directory of the resolved path does not exist.
+ */
+export function resolveOutputPath(outputPath, cwd) {
+    const resolved = path.isAbsolute(outputPath)
+        ? outputPath
+        : path.resolve(cwd, outputPath);
+    const ext = `.${Archive.FILE_EXTENSION}`;
+    const withExt = resolved.endsWith(ext) ? resolved : `${resolved}${ext}`;
+    const dir = path.dirname(withExt);
+    if (!existsSync(dir)) {
+        throw new Error(`Output directory does not exist: ${dir}. Please create the directory before running the command.`);
+    }
+    return withExt;
+}

package/lib/types.d.ts

@@ -1,4 +1,4 @@
-import type { CrawlerError } from './utils/index.js';
+import type { CrawlerError } from './utils/types/types.js';
 /**
  * Event map for the `CrawlerOrchestrator` class.
  *

package/lib/utils/types/types.d.ts

@@ -41,6 +41,8 @@ export interface CrawlerError {
     isMainProcess: boolean;
     /** The URL being processed when the error occurred, or `null` if not applicable. */
     url: string | null;
+    /** Whether the error occurred while processing an external (out-of-scope) URL. */
+    isExternal: boolean;
     /** The error object. */
     error: Error;
 }

package/lib/write-queue.d.ts

@@ -0,0 +1,32 @@
+/**
+ * A lightweight serial write queue that ensures asynchronous operations
+ * execute one at a time in FIFO order.
+ *
+ * Used to serialize concurrent Archive writes from crawler event handlers,
+ * preventing SQLite write-lock contention under high parallelism.
+ */
+export declare class WriteQueue {
+    #private;
+    /**
+     * The number of operations currently waiting or executing in the queue.
+     * @returns The current pending operation count.
+     */
+    get pending(): number;
+    /**
+     * Returns a promise that resolves when all currently enqueued operations
+     * have completed.
+     * @returns A promise that resolves when the queue is drained.
+     */
+    drain(): Promise<void>;
+    /**
+     * Enqueues an operation to run after all previously enqueued operations
+     * have completed. Operations are guaranteed to execute serially in the
+     * order they were enqueued. Both async and synchronous (throwing)
+     * operations are supported.
+     * @template T The resolved type of the operation's promise.
+     * @param operation - The function to execute. May return a `Promise` or a plain value.
+     * @returns A promise that resolves with the operation's result, or rejects
+     *          if the operation throws.
+     */
+    enqueue<T>(operation: () => Promise<T> | T): Promise<T>;
+}

package/lib/write-queue.js

@@ -0,0 +1,55 @@
+/**
+ * A lightweight serial write queue that ensures asynchronous operations
+ * execute one at a time in FIFO order.
+ *
+ * Used to serialize concurrent Archive writes from crawler event handlers,
+ * preventing SQLite write-lock contention under high parallelism.
+ */
+export class WriteQueue {
+    /** The tail of the promise chain used for serialization. */
+    #chain = Promise.resolve();
+    /** The number of operations currently waiting or executing in the queue. */
+    #pending = 0;
+    /**
+     * The number of operations currently waiting or executing in the queue.
+     * @returns The current pending operation count.
+     */
+    get pending() {
+        return this.#pending;
+    }
+    /**
+     * Returns a promise that resolves when all currently enqueued operations
+     * have completed.
+     * @returns A promise that resolves when the queue is drained.
+     */
+    async drain() {
+        await this.#chain;
+    }
+    /**
+     * Enqueues an operation to run after all previously enqueued operations
+     * have completed. Operations are guaranteed to execute serially in the
+     * order they were enqueued. Both async and synchronous (throwing)
+     * operations are supported.
+     * @template T The resolved type of the operation's promise.
+     * @param operation - The function to execute. May return a `Promise` or a plain value.
+     * @returns A promise that resolves with the operation's result, or rejects
+     *          if the operation throws.
+     */
+    enqueue(operation) {
+        this.#pending++;
+        return new Promise((resolve, reject) => {
+            this.#chain = this.#chain.then(async () => {
+                try {
+                    const result = await operation();
+                    resolve(result);
+                }
+                catch (error) {
+                    reject(error);
+                }
+                finally {
+                    this.#pending--;
+                }
+            });
+        });
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@nitpicker/crawler",
-	"version": "0.4.4",
+	"version": "0.6.0",
 	"description": "Web crawler engine with headless browser rendering and archive storage",
 	"author": "D-ZERO",
 	"license": "Apache-2.0",
@@ -18,8 +18,8 @@
 	"type": "module",
 	"exports": {
 		".": {
-			"import": "./lib/index.js",
-			"types": "./lib/index.d.ts"
+			"import": "./lib/crawler.js",
+			"types": "./lib/crawler.d.ts"
 		}
 	},
 	"scripts": {
@@ -28,9 +28,9 @@
 	},
 	"dependencies": {
 		"@d-zero/beholder": "2.0.0",
-		"@d-zero/dealer": "1.6.3",
+		"@d-zero/dealer": "1.7.0",
 		"@d-zero/fs": "0.2.2",
-		"@d-zero/shared": "0.20.0",
+		"@d-zero/shared": "0.20.1",
 		"ansi-colors": "4.1.3",
 		"debug": "4.4.3",
 		"follow-redirects": "1.15.11",
@@ -48,5 +48,5 @@
 		"@types/tar": "7.0.87",
 		"@types/unzipper": "0.10.11"
 	},
-	"gitHead": "7c9bad351d1a3f4eb6ea7233f7143140d0989ef2"
+	"gitHead": "eab407f5e4b58fa3c122001d3c034488e7f6da11"
 }