@nitpicker/crawler 0.4.1

package/lib/crawler/speculative-pagination.js

@@ -0,0 +1,215 @@
+import { isError } from '@nitpicker/beholder';
+/**
+ * Compares two consecutive URL strings and detects a single-token numeric
+ * pagination pattern (e.g. `/page/1` → `/page/2`, or `?p=1` → `?p=2`).
+ *
+ * The algorithm decomposes each URL into tokens (path segments + sorted query values),
+ * then checks that exactly one token differs and both values are integers with a
+ * positive step. Returns `null` when no pattern is detected.
+ *
+ * WHY single-token constraint: Multi-token differences (e.g. both path and query
+ * changing) indicate different routes rather than pagination, so they are rejected.
+ * @param prevUrl - The previously pushed URL (protocol-agnostic, without hash/auth)
+ * @param currentUrl - The newly discovered URL
+ * @returns The detected pattern, or `null` if no pagination pattern was found
+ */
+export function detectPaginationPattern(prevUrl, currentUrl) {
+    const prev = decomposeUrl(prevUrl);
+    const curr = decomposeUrl(currentUrl);
+    if (!prev || !curr)
+        return null;
+    // Host (including port) must match
+    if (prev.host !== curr.host)
+        return null;
+    // Path segment count must match
+    if (prev.pathSegments.length !== curr.pathSegments.length)
+        return null;
+    // Query key sets must match in count and identity
+    if (prev.queryKeys.length !== curr.queryKeys.length)
+        return null;
+    for (let i = 0; i < prev.queryKeys.length; i++) {
+        if (prev.queryKeys[i] !== curr.queryKeys[i])
+            return null;
+    }
+    // Build combined token arrays: path segments + query values (sorted by key)
+    const prevTokens = [...prev.pathSegments, ...prev.queryValues];
+    const currTokens = [...curr.pathSegments, ...curr.queryValues];
+    let diffIndex = -1;
+    for (const [i, prevToken] of prevTokens.entries()) {
+        if (prevToken !== currTokens[i]) {
+            if (diffIndex !== -1)
+                return null; // more than one difference
+            diffIndex = i;
+        }
+    }
+    if (diffIndex === -1)
+        return null; // identical URLs
+    const prevNum = Number(prevTokens[diffIndex]);
+    const currNum = Number(currTokens[diffIndex]);
+    if (!Number.isFinite(prevNum) || !Number.isFinite(currNum))
+        return null;
+    if (!Number.isInteger(prevNum) || !Number.isInteger(currNum))
+        return null;
+    const step = currNum - prevNum;
+    if (step <= 0)
+        return null;
+    return {
+        tokenIndex: diffIndex,
+        step,
+        currentNumber: currNum,
+    };
+}
+/**
+ * Generates speculative URLs by extrapolating the detected pagination pattern.
+ *
+ * Starting from `currentUrl`, applies the pattern's step `count` times to produce
+ * future page URLs (e.g. if step=1 and currentNumber=2, generates page 3, 4, ...).
+ * These URLs are pushed into the crawl queue and discarded later if they 404.
+ * @param pattern - The detected pagination pattern from {@link detectPaginationPattern}
+ * @param currentUrl - The URL to extrapolate from (protocol-agnostic, without hash/auth)
+ * @param count - Number of speculative URLs to generate (typically equals concurrency)
+ * @returns Array of speculative URL strings
+ */
+export function generateSpeculativeUrls(pattern, currentUrl, count) {
+    if (count <= 0)
+        return [];
+    const decomposed = decomposeUrl(currentUrl);
+    if (!decomposed)
+        return [];
+    const results = [];
+    for (let i = 1; i <= count; i++) {
+        const nextNum = pattern.currentNumber + pattern.step * i;
+        const url = reconstructUrl(decomposed, pattern.tokenIndex, String(nextNum));
+        results.push(url);
+    }
+    return results;
+}
+/**
+ * Determines whether a speculative URL's scrape result should be discarded.
+ *
+ * Speculative URLs are pre-emptively pushed into the crawl queue before
+ * knowing if they exist. This function filters out invalid results:
+ * - `error` type → discard (server unreachable, timeout, etc.)
+ * - `ignoreAndSkip` type → discard (matched exclusion rule)
+ * - `scrapeEnd` with HTTP error status (4xx/5xx) → discard
+ * - `scrapeEnd` with 2xx/3xx → keep
+ * @param result - The scrape result for the speculative URL
+ * @returns `true` if the result should be discarded (not saved to archive)
+ */
+export function shouldDiscardSpeculative(result) {
+    switch (result.type) {
+        case 'error': {
+            return true;
+        }
+        case 'ignoreAndSkip': {
+            return true;
+        }
+        case 'scrapeEnd': {
+            if (!result.pageData)
+                return true;
+            return isError(result.pageData.status);
+        }
+        default: {
+            return true;
+        }
+    }
+}
+/**
+ * Decomposes a URL string into its constituent tokens for comparison.
+ * Handles both full URLs (`https://host/path?q=v`) and protocol-agnostic
+ * URLs (`//host/path?q=v`). Query parameters are sorted by key for
+ * consistent comparison.
+ * @param url - The URL string to decompose
+ * @returns The decomposed URL, or `null` if the format is invalid
+ */
+function decomposeUrl(url) {
+    // URL format: //host/path?query  or  //host?query  (protocol-agnostic)
+    // Also handle protocol://host/path?query
+    let work = url;
+    let protocol = '';
+    // Strip protocol
+    const protoMatch = /^(https?:)?\/\//.exec(work);
+    if (!protoMatch)
+        return null;
+    protocol = protoMatch[1] ?? '';
+    work = work.slice(protoMatch[0].length);
+    // Split host from rest
+    const slashIdx = work.indexOf('/');
+    const qmarkIdx = work.indexOf('?');
+    let host;
+    let pathPart;
+    let queryPart;
+    if (slashIdx === -1 && qmarkIdx === -1) {
+        host = work;
+        pathPart = '';
+        queryPart = '';
+    }
+    else if (slashIdx === -1) {
+        host = work.slice(0, qmarkIdx);
+        pathPart = '';
+        queryPart = work.slice(qmarkIdx + 1);
+    }
+    else {
+        host = work.slice(0, slashIdx);
+        const pathAndQuery = work.slice(slashIdx + 1);
+        const pq = pathAndQuery.indexOf('?');
+        if (pq === -1) {
+            pathPart = pathAndQuery;
+            queryPart = '';
+        }
+        else {
+            pathPart = pathAndQuery.slice(0, pq);
+            queryPart = pathAndQuery.slice(pq + 1);
+        }
+    }
+    const pathSegments = pathPart ? pathPart.split('/') : [];
+    // Parse query into sorted key-value pairs
+    const queryPairs = [];
+    if (queryPart) {
+        for (const pair of queryPart.split('&')) {
+            const eqIdx = pair.indexOf('=');
+            if (eqIdx === -1) {
+                queryPairs.push([pair, '']);
+            }
+            else {
+                queryPairs.push([pair.slice(0, eqIdx), pair.slice(eqIdx + 1)]);
+            }
+        }
+    }
+    queryPairs.sort((a, b) => a[0].localeCompare(b[0]));
+    return {
+        host,
+        pathSegments,
+        queryKeys: queryPairs.map(([k]) => k),
+        queryValues: queryPairs.map(([, v]) => v),
+        protocol,
+    };
+}
+/**
+ * Reconstructs a URL string from a decomposed representation with one
+ * token replaced at the specified index.
+ * @param decomposed - The decomposed URL to reconstruct
+ * @param tokenIndex - Index in the combined token array (path segments + query values)
+ * @param newValue - The replacement value for the token at `tokenIndex`
+ * @returns The reconstructed URL string
+ */
+function reconstructUrl(decomposed, tokenIndex, newValue) {
+    const { host, pathSegments, queryKeys, queryValues, protocol } = decomposed;
+    const newPathSegments = [...pathSegments];
+    const newQueryValues = [...queryValues];
+    if (tokenIndex < pathSegments.length) {
+        newPathSegments[tokenIndex] = newValue;
+    }
+    else {
+        newQueryValues[tokenIndex - pathSegments.length] = newValue;
+    }
+    let url = `${protocol}//${host}`;
+    if (newPathSegments.length > 0) {
+        url += `/${newPathSegments.join('/')}`;
+    }
+    if (queryKeys.length > 0) {
+        const pairs = queryKeys.map((k, i) => `${k}=${newQueryValues[i]}`);
+        url += `?${pairs.join('&')}`;
+    }
+    return url;
+}

package/lib/crawler/types.d.ts

@@ -0,0 +1,119 @@
+import type { PageData, CrawlerError, Resource } from '../utils/index.js';
+import type { ChangePhaseEvent } from '@d-zero/beholder';
+import type { ParseURLOptions } from '@d-zero/shared/parse-url';
+/**
+ * Configuration options that control crawler behavior.
+ *
+ * Used by the result handler functions to determine how to process
+ * scrape results, which URLs to follow, and how to handle external links.
+ * @see {@link ./crawler.ts | Crawler} for the main consumer of this type
+ * @see {@link ../crawler-orchestrator.ts | CrawlerOrchestrator} for factory methods that build these options
+ */
+export interface CrawlerOptions extends Required<Pick<ParseURLOptions, 'disableQueries'>> {
+    /** Delay in milliseconds between page requests. */
+    interval: number;
+    /** Maximum number of concurrent scraping processes. 0 uses the default. */
+    parallels: number;
+    /** Whether to recursively follow discovered links within the scope. */
+    recursive: boolean;
+    /** Whether the crawl was started from a pre-defined URL list. */
+    fromList: boolean;
+    /** Whether to capture image resources during scraping. */
+    captureImages: boolean;
+    /** Path to the Chromium/Chrome executable, or `null` for the bundled version. */
+    executablePath: string | null;
+    /** Whether to fetch and scrape external (out-of-scope) pages. */
+    fetchExternal: boolean;
+    /** List of scope URL strings that define the crawl boundary. */
+    scope: string[];
+    /** Glob patterns for URLs to exclude from crawling. */
+    excludes: string[];
+    /** Keywords that trigger page exclusion when found in content. */
+    excludeKeywords: string[];
+    /** URL prefixes to exclude from crawling (merged defaults + user additions). */
+    excludeUrls: readonly string[];
+    /** Maximum directory depth for excluded paths. */
+    maxExcludedDepth: number;
+    /** Maximum number of retry attempts per URL on scrape failure. */
+    retry: number;
+    /** Whether to enable verbose logging. */
+    verbose: boolean;
+    /** User-Agent string sent with HTTP requests. */
+    userAgent: string;
+    /** Whether to ignore robots.txt restrictions. */
+    ignoreRobots: boolean;
+}
+/**
+ * Describes a detected pagination pattern between two consecutive URLs.
+ */
+export interface PaginationPattern {
+    /** Index within the combined token array (path segments + query values) where the numeric difference was found. */
+    tokenIndex: number;
+    /** The numeric increment (always > 0). */
+    step: number;
+    /** The number found at `tokenIndex` in the "current" URL. */
+    currentNumber: number;
+}
+/**
+ * Event map for the `Crawler` class.
+ *
+ * Each key represents an event name and its value is the payload type
+ * passed to listeners subscribed via `on()` or `once()`.
+ */
+export interface CrawlerEventTypes {
+    /**
+     * Emitted when a page within the crawl scope has been successfully scraped.
+     */
+    page: {
+        /** The scraped page data including HTML, metadata, anchors, and images. */
+        result: PageData;
+    };
+    /**
+     * Emitted when an external page (outside the crawl scope) has been scraped.
+     */
+    externalPage: {
+        /** The scraped page data for the external page. */
+        result: PageData;
+    };
+    /**
+     * Emitted when a URL is skipped due to exclusion rules, robots.txt restrictions,
+     * or external fetch being disabled.
+     */
+    skip: {
+        /** The URL that was skipped. */
+        url: string;
+        /** The reason the URL was skipped (e.g., "excluded", "blocked by robots.txt", or a JSON description). */
+        reason: string;
+        /** Whether the skipped URL is external to the crawl scope. */
+        isExternal: boolean;
+    };
+    /**
+     * Emitted when a network resource (CSS, JS, image, etc.) is captured during page scraping.
+     */
+    response: {
+        /** The captured resource data including URL, status, content type, and headers. */
+        resource: Resource;
+    };
+    /**
+     * Emitted to record the relationship between a page and a resource it references.
+     */
+    responseReferrers: {
+        /** The URL of the page that references the resource. */
+        url: string;
+        /** The URL of the referenced resource (without hash). */
+        src: string;
+    };
+    /**
+     * Emitted when the entire crawl process has completed or been aborted.
+     */
+    crawlEnd: Record<string, unknown>;
+    /**
+     * Emitted when an error occurs during crawling.
+     */
+    error: CrawlerError;
+    /**
+     * Emitted when the scraper transitions between phases of the page scraping lifecycle
+     * (e.g., scrapeStart, headRequest, openPage, success).
+     */
+    changePhase: ChangePhaseEvent;
+}

package/lib/crawler/types.js

@@ -0,0 +1 @@
+export {};

package/lib/crawler/url-filter.d.ts

@@ -0,0 +1,56 @@
+import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
+/**
+ * Determine whether a URL should be skipped during crawling.
+ *
+ * A URL is skipped if it matches any user-defined exclude glob pattern
+ * or starts with any of the excluded URL prefixes.
+ * @param url - The parsed URL to check.
+ * @param excludes - Array of glob patterns for URLs to exclude.
+ * @param excludeUrls - Array of URL prefixes to exclude (matched via `startsWith`).
+ * @param options - URL parsing options used for pattern matching.
+ * @returns `true` if the URL should be skipped.
+ */
+export declare function shouldSkipUrl(url: ExURL, excludes: readonly string[], excludeUrls: readonly string[], options: ParseURLOptions): boolean;
+/**
+ * Determine whether a URL is external to the crawl scope.
+ *
+ * A URL is considered external if its hostname does not appear
+ * as a key in the scope map.
+ * @param url - The parsed URL to check.
+ * @param scope - Map of hostnames to their scope URLs.
+ * @returns `true` if the URL is outside the crawl scope.
+ */
+export declare function isExternalUrl(url: ExURL, scope: ReadonlyMap<string, readonly ExURL[]>): boolean;
+/**
+ * Inject authentication credentials from a matching scope URL into the target URL.
+ *
+ * Finds the best-matching scope URL (deepest path match) for the given URL's
+ * hostname and copies its `username` and `password` properties. This mutates
+ * the `url` parameter in place.
+ * @param url - The parsed URL to receive authentication credentials (mutated in place).
+ * @param scope - Map of hostnames to their scope URLs.
+ */
+export declare function injectScopeAuth(url: ExURL, scope: ReadonlyMap<string, readonly ExURL[]>): void;
+/**
+ * Find the scope URL with the deepest matching path for a given URL.
+ *
+ * Among all scope URLs sharing the same hostname, returns the one whose
+ * path segments are a prefix of the target URL's path segments and which
+ * has the greatest depth. Returns `null` if no scope URL matches.
+ * @param url - The parsed URL to match against scope URLs.
+ * @param scopes - The list of scope URLs to search.
+ * @returns The best-matching scope URL, or `null` if none match.
+ */
+export declare function findBestMatchingScope(url: ExURL, scopes: readonly ExURL[]): ExURL | null;
+/**
+ * Check whether a URL is in a lower layer (subdirectory) of any scope URL.
+ *
+ * Tests the URL against each scope URL using the `isLowerLayer` utility,
+ * which checks if the URL's path is at the same level or deeper than
+ * the scope URL's path.
+ * @param url - The parsed URL to check.
+ * @param scopes - The list of scope URLs to test against.
+ * @param options - URL parsing options used for layer comparison.
+ * @returns `true` if the URL is in a lower layer of at least one scope URL.
+ */
+export declare function isInAnyLowerLayer(url: ExURL, scopes: readonly ExURL[], options: ParseURLOptions): boolean;

package/lib/crawler/url-filter.js

@@ -0,0 +1,110 @@
+import { isLowerLayer } from '@d-zero/shared/is-lower-layer';
+import { pathMatch } from '@d-zero/shared/path-match';
+import { protocolAgnosticKey } from './protocol-agnostic-key.js';
+/**
+ * Determine whether a URL should be skipped during crawling.
+ *
+ * A URL is skipped if it matches any user-defined exclude glob pattern
+ * or starts with any of the excluded URL prefixes.
+ * @param url - The parsed URL to check.
+ * @param excludes - Array of glob patterns for URLs to exclude.
+ * @param excludeUrls - Array of URL prefixes to exclude (matched via `startsWith`).
+ * @param options - URL parsing options used for pattern matching.
+ * @returns `true` if the URL should be skipped.
+ */
+export function shouldSkipUrl(url, excludes, excludeUrls, options) {
+    return (excludes.some((excludeGlobPattern) => pathMatch(url, excludeGlobPattern, options)) ||
+        excludeUrls.some((prefix) => protocolAgnosticKey(url.href).startsWith(protocolAgnosticKey(prefix))));
+}
+/**
+ * Determine whether a URL is external to the crawl scope.
+ *
+ * A URL is considered external if its hostname does not appear
+ * as a key in the scope map.
+ * @param url - The parsed URL to check.
+ * @param scope - Map of hostnames to their scope URLs.
+ * @returns `true` if the URL is outside the crawl scope.
+ */
+export function isExternalUrl(url, scope) {
+    return !scope.has(url.hostname);
+}
+/**
+ * Inject authentication credentials from a matching scope URL into the target URL.
+ *
+ * Finds the best-matching scope URL (deepest path match) for the given URL's
+ * hostname and copies its `username` and `password` properties. This mutates
+ * the `url` parameter in place.
+ * @param url - The parsed URL to receive authentication credentials (mutated in place).
+ * @param scope - Map of hostnames to their scope URLs.
+ */
+export function injectScopeAuth(url, scope) {
+    const scopes = scope.get(url.hostname);
+    if (!scopes) {
+        return;
+    }
+    const matchedScope = findBestMatchingScope(url, scopes);
+    if (matchedScope) {
+        url.username = matchedScope.username;
+        url.password = matchedScope.password;
+    }
+}
+/**
+ * Find the scope URL with the deepest matching path for a given URL.
+ *
+ * Among all scope URLs sharing the same hostname, returns the one whose
+ * path segments are a prefix of the target URL's path segments and which
+ * has the greatest depth. Returns `null` if no scope URL matches.
+ * @param url - The parsed URL to match against scope URLs.
+ * @param scopes - The list of scope URLs to search.
+ * @returns The best-matching scope URL, or `null` if none match.
+ */
+export function findBestMatchingScope(url, scopes) {
+    let bestMatch = null;
+    let maxDepth = -1;
+    for (const scope of scopes) {
+        if (url.hostname !== scope.hostname) {
+            continue;
+        }
+        const isMatch = isPathMatch(url.paths, scope.paths);
+        if (isMatch && scope.depth > maxDepth) {
+            bestMatch = scope;
+            maxDepth = scope.depth;
+        }
+    }
+    return bestMatch;
+}
+/**
+ * Check whether a target path is equal to or is a descendant of a base path.
+ *
+ * Compares path segments element by element. The target path matches if
+ * all segments of the base path appear in the same positions at the
+ * beginning of the target path.
+ * @param targetPaths - The path segments of the URL being checked.
+ * @param basePaths - The path segments of the scope URL to match against.
+ * @returns `true` if the target path starts with or equals the base path.
+ */
+function isPathMatch(targetPaths, basePaths) {
+    if (targetPaths.length < basePaths.length) {
+        return false;
+    }
+    for (const [i, basePath] of basePaths.entries()) {
+        if (targetPaths[i] !== basePath) {
+            return false;
+        }
+    }
+    return true;
+}
+/**
+ * Check whether a URL is in a lower layer (subdirectory) of any scope URL.
+ *
+ * Tests the URL against each scope URL using the `isLowerLayer` utility,
+ * which checks if the URL's path is at the same level or deeper than
+ * the scope URL's path.
+ * @param url - The parsed URL to check.
+ * @param scopes - The list of scope URLs to test against.
+ * @param options - URL parsing options used for layer comparison.
+ * @returns `true` if the URL is in a lower layer of at least one scope URL.
+ */
+export function isInAnyLowerLayer(url, scopes, options) {
+    return scopes.some((scope) => isLowerLayer(url.href, scope.href, options));
+}

package/lib/crawler-orchestrator.d.ts

@@ -0,0 +1,142 @@
+import type { Config } from './archive/types.js';
+import type { CrawlEvent } from './types.js';
+import type { ExURL } from '@d-zero/shared/parse-url';
+import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
+import Archive from './archive/archive.js';
+/**
+ * Default list of external URL prefixes excluded from crawling.
+ * Includes social media sharing endpoints that are commonly linked
+ * but provide no useful crawl data.
+ */
+export declare const DEFAULT_EXCLUDED_EXTERNAL_URLS: string[];
+/**
+ * Configuration options for the CrawlerOrchestrator.
+ *
+ * Extends the archive {@link Config} with additional runtime settings
+ * such as working directory, browser executable path, and output options.
+ */
+interface CrawlConfig extends Config {
+    /** The working directory for output files. Defaults to `process.cwd()`. */
+    cwd: string;
+    /** Path to a Chromium/Chrome executable for Puppeteer. */
+    executablePath: string;
+    /** Output file path for the archive. */
+    filePath: string;
+    /** Whether to capture image resources during crawling. */
+    image: boolean;
+    /** File-size threshold (in bytes) above which images are excluded. */
+    imageFileSizeThreshold: number;
+    /** Delay in milliseconds between each page request. */
+    interval: number;
+    /** Whether the input is a pre-defined URL list (non-recursive mode). */
+    list: boolean;
+    /** Maximum number of retry attempts per URL on scrape failure. */
+    retry: number;
+    /** Whether to enable verbose logging output. */
+    verbose: boolean;
+    /** Custom User-Agent string for HTTP requests. */
+    userAgent: string;
+    /** Whether to ignore robots.txt restrictions. */
+    ignoreRobots: boolean;
+}
+/**
+ * Callback invoked after the CrawlerOrchestrator instance is fully initialized
+ * but before crawling begins.
+ * @param orchestrator - The initialized CrawlerOrchestrator instance.
+ * @param config - The resolved archive configuration.
+ */
+type CrawlInitializedCallback = (orchestrator: CrawlerOrchestrator, config: Config) => void | Promise<void>;
+/**
+ * The main entry point for Nitpicker web crawling and archiving.
+ *
+ * CrawlerOrchestrator orchestrates the full lifecycle of a crawl session: it creates an archive,
+ * configures a {@link Crawler}, processes discovered pages and resources, and
+ * writes the final archive file. It emits events defined by {@link CrawlEvent}.
+ *
+ * Instances are created via the static factory methods {@link CrawlerOrchestrator.crawling}
+ * or {@link CrawlerOrchestrator.resume}; the constructor is private.
+ * @example
+ * ```ts
+ * const orchestrator = await CrawlerOrchestrator.crawling(['https://example.com'], { recursive: true });
+ * await orchestrator.write();
+ * ```
+ */
+export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
+    #private;
+    /**
+     * The underlying archive instance used for storing crawl results.
+     */
+    get archive(): Archive;
+    private constructor();
+    /**
+     * Abort the current crawl and archive operations.
+     *
+     * Delegates to the archive's abort method, which stops all in-progress
+     * database writes and cleans up temporary resources.
+     * @returns The result of the archive abort operation.
+     */
+    abort(): void;
+    /**
+     * Execute the crawl for the given list of URLs.
+     *
+     * Sets up event listeners on the crawler, starts crawling, and resolves
+     * when the crawl completes. Discovered pages, external pages, skipped pages,
+     * and resources are forwarded to the archive for storage.
+     * @param list - The list of parsed URLs to crawl. The first URL is used as the root.
+     * @returns A promise that resolves when crawling is complete.
+     * @throws {Error} If the URL list is empty.
+     */
+    crawling(list: ExURL[]): Promise<void>;
+    /**
+     * Kill any zombie Chromium processes that were not properly cleaned up.
+     *
+     * Retrieves the list of undead process IDs from the crawler and sends
+     * a SIGTERM signal to each one. Chromium is intentionally sent SIGTERM
+     * (not SIGKILL) to avoid leaving zombie processes.
+     */
+    garbageCollect(): void;
+    /**
+     * Retrieve the list of process IDs for Chromium instances that are
+     * still running after crawling has ended.
+     * @returns An array of process IDs that should be terminated.
+     */
+    getUndeadPid(): never[];
+    /**
+     * Write the archive to its configured file path.
+     *
+     * Emits `writeFileStart` before writing and `writeFileEnd` after
+     * the write completes successfully.
+     */
+    write(): Promise<void>;
+    /**
+     * Create a new CrawlerOrchestrator instance and start crawling the given URLs.
+     *
+     * This is the primary factory method for starting a fresh crawl. It:
+     * 1. Parses and sorts the input URLs
+     * 2. Creates an archive file
+     * 3. Saves the crawl configuration
+     * 4. Runs the optional initialized callback
+     * 5. Executes the crawl
+     * 6. Sorts the archived URLs in natural order
+     * @param url - One or more URL strings to crawl.
+     * @param options - Optional configuration overrides for the crawl session.
+     * @param initializedCallback - Optional callback invoked after initialization but before crawling starts.
+     * @returns A promise that resolves to the CrawlerOrchestrator instance after crawling completes.
+     * @throws {Error} If the URL list is empty or contains no valid URLs.
+     */
+    static crawling(url: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback): Promise<CrawlerOrchestrator>;
+    /**
+     * Resume a previously interrupted crawl from an existing archive file.
+     *
+     * Restores the crawl state (pending URLs, scraped URLs, and resources)
+     * from the archive, merges any option overrides, and continues crawling
+     * from where it left off.
+     * @param stubPath - Path to the existing archive file to resume from.
+     * @param options - Optional configuration overrides to apply on top of the archived config.
+     * @param initializedCallback - Optional callback invoked after initialization but before crawling resumes.
+     * @returns A promise that resolves to the CrawlerOrchestrator instance after crawling completes.
+     * @throws {Error} If the archived URL is invalid.
+     */
+    static resume(stubPath: string, options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback): Promise<CrawlerOrchestrator>;
+}
+export {};