@nitpicker/crawler 0.4.1

package/lib/crawler/crawler.d.ts

@@ -0,0 +1,73 @@
+import type { CrawlerEventTypes, CrawlerOptions } from './types.js';
+import type { ExURL } from '@d-zero/shared/parse-url';
+import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
+export type { CrawlerOptions } from './types.js';
+/**
+ * The core crawler engine that discovers and scrapes web pages.
+ *
+ * The Crawler manages the crawl queue, uses the dealer pattern for concurrent
+ * page scraping via `@d-zero/beholder`, handles scrape results, and emits
+ * events defined by {@link CrawlerEventTypes}. It supports recursive crawling
+ * within a defined scope, external page fetching, URL exclusion, and resumable crawls.
+ *
+ * Crawling is performed concurrently using the dealer pattern, with
+ * configurable parallelism up to {@link Crawler.MAX_PROCESS_LENGTH}.
+ */
+export default class Crawler extends EventEmitter<CrawlerEventTypes> {
+    #private;
+    /**
+     * Create a new Crawler instance.
+     * @param options - Configuration options for crawling behavior. All fields have
+     *   sensible defaults if omitted.
+     */
+    constructor(options?: Partial<CrawlerOptions>);
+    /**
+     * Abort the current crawl operation.
+     *
+     * Sets the aborted flag and immediately emits a `crawlEnd` event.
+     * In-progress scrape tasks will check the flag and exit early.
+     */
+    abort(): void;
+    /**
+     * Retrieve the list of Chromium process IDs that are still running.
+     *
+     * In the current architecture, process cleanup is handled by the dealer,
+     * so this always returns an empty array.
+     * @returns An empty array (reserved for future use).
+     */
+    getUndeadPid(): never[];
+    /**
+     * Restore crawl state from a previous session for resumable crawling.
+     *
+     * Repopulates the link list with pending and already-scraped URLs,
+     * and restores the set of known resource URLs to avoid duplicates.
+     * @param pending - URLs that were pending (not yet scraped) in the previous session.
+     * @param scraped - URLs that were already scraped in the previous session.
+     * @param resources - Resource URLs that were already captured in the previous session.
+     */
+    resume(pending: string[], scraped: string[], resources: string[]): void;
+    /**
+     * Start crawling from a single root URL.
+     *
+     * Adds the root URL to the scope (if not already present) and the link list,
+     * then begins the deal-based concurrent crawl. Discovered child pages are
+     * automatically added to the queue when recursive mode is enabled.
+     * @param url - The root URL to begin crawling from.
+     */
+    start(url: ExURL): void;
+    /**
+     * Start crawling a pre-defined list of URLs in non-recursive mode.
+     *
+     * Each URL in the list is added to the scope and the link list. Recursive
+     * crawling is disabled; only the provided URLs will be scraped.
+     * @param pageList - The list of URLs to crawl. Must contain at least one URL.
+     * @throws {Error} If the page list is empty.
+     */
+    startMultiple(pageList: ExURL[]): void;
+    /**
+     * The default maximum number of concurrent scraping processes.
+     *
+     * Used when `parallels` is not specified or is set to 0.
+     */
+    static readonly MAX_PROCESS_LENGTH = 10;
+}