@crawlee/basic 4.0.0-beta.170 → 4.0.0-beta.172

package/internals/cookie_utils.js

@@ -0,0 +1,122 @@
+import { serviceLocator } from '@crawlee/core';
+import { Cookie, CookieJar } from 'tough-cookie';
+import { CookieParseError } from './session_pool/errors.js';
+/**
+ * @internal
+ */
+export function getCookiesFromResponse(response) {
+    const headers = response.headers;
+    const cookieHeaders = headers.getSetCookie();
+    try {
+        return cookieHeaders.map((cookie) => Cookie.parse(cookie));
+    }
+    catch (e) {
+        throw new CookieParseError(cookieHeaders);
+    }
+}
+/**
+ * Calculate cookie expiration date
+ * @param maxAgeSecs
+ * @returns Calculated date by session max age seconds.
+ * @internal
+ */
+export function getDefaultCookieExpirationDate(maxAgeSecs) {
+    return new Date(Date.now() + maxAgeSecs * 1000);
+}
+/**
+ * Transforms tough-cookie to puppeteer cookie.
+ * @param toughCookie Cookie from CookieJar
+ * @return Cookie compatible with browser pool
+ * @internal
+ */
+export function toughCookieToBrowserPoolCookie(toughCookie) {
+    return {
+        name: toughCookie.key,
+        value: toughCookie.value,
+        // Puppeteer and Playwright expect 'expires' to be 'Unix time in seconds', not ms
+        // If there is no expires date (so defaults to Infinity), we don't provide it to the browsers
+        expires: toughCookie.expires == null || toughCookie.expires === 'Infinity'
+            ? undefined
+            : toughCookie.expires.getTime() / 1000,
+        domain: toughCookie.domain ? `${toughCookie.hostOnly ? '' : '.'}${toughCookie.domain}` : undefined,
+        path: toughCookie.path ?? undefined,
+        secure: toughCookie.secure,
+        httpOnly: toughCookie.httpOnly,
+    };
+}
+/**
+ * Transforms browser-pool cookie to tough-cookie.
+ * @param cookieObject Cookie object (for instance from the `page.cookies` method).
+ * @param maxAgeSecs Fallback expiration in seconds when the cookie itself has no `expires`.
+ * When omitted, such a cookie is stored as a session cookie (no automatic expiration).
+ * @internal
+ */
+export function browserPoolCookieToToughCookie(cookieObject, maxAgeSecs) {
+    const isExpiresValid = cookieObject.expires && typeof cookieObject.expires === 'number' && cookieObject.expires > 0;
+    let expires;
+    if (isExpiresValid) {
+        expires = new Date(cookieObject.expires * 1000);
+    }
+    else if (maxAgeSecs != null) {
+        expires = getDefaultCookieExpirationDate(maxAgeSecs);
+    }
+    const domainHasLeadingDot = cookieObject.domain?.startsWith?.('.');
+    const domain = domainHasLeadingDot ? cookieObject.domain?.slice?.(1) : cookieObject.domain;
+    return new Cookie({
+        key: cookieObject.name,
+        value: cookieObject.value,
+        ...(expires !== undefined && { expires }),
+        domain,
+        path: cookieObject.path,
+        secure: cookieObject.secure,
+        httpOnly: cookieObject.httpOnly,
+        hostOnly: !domainHasLeadingDot,
+    });
+}
+/**
+ * @internal
+ * @param cookieString The cookie string to attempt parsing
+ * @returns Browser pool compatible cookie, or null if cookie cannot be parsed
+ */
+export function cookieStringToToughCookie(cookieString) {
+    const parsed = Cookie.parse(cookieString);
+    if (parsed) {
+        return toughCookieToBrowserPoolCookie(parsed);
+    }
+    return null;
+}
+/**
+ * Merges multiple cookie strings. Keys are compared case-sensitively, warning will be logged
+ * if we see two cookies with same keys but different casing.
+ * @internal
+ */
+export function mergeCookies(url, sourceCookies) {
+    const jar = new CookieJar();
+    // ignore empty cookies
+    for (const sourceCookieString of sourceCookies) {
+        // ignore empty cookies
+        if (!sourceCookieString)
+            continue;
+        const cookies = sourceCookieString.split(/ *; */);
+        for (const cookieString of cookies) {
+            // ignore extra spaces
+            if (!cookieString)
+                continue;
+            const cookie = Cookie.parse(cookieString);
+            if (!cookie) {
+                serviceLocator.getLogger().warning(`Skipping malformed cookie fragment: '${cookieString}'`);
+                continue;
+            }
+            const similarKeyCookie = jar.getCookiesSync(url).find((c) => {
+                return cookie.key !== c.key && cookie.key.toLowerCase() === c.key.toLowerCase();
+            });
+            if (similarKeyCookie) {
+                serviceLocator
+                    .getLogger()
+                    .warningOnce(`Found cookies with similar name during cookie merging: '${cookie.key}' and '${similarKeyCookie.key}'`);
+            }
+            jar.setCookieSync(cookie, url);
+        }
+    }
+    return jar.getCookieStringSync(url);
+}

package/internals/crawler-run.d.ts

@@ -1,4 +1,6 @@
-import type { AutoscaledPoolOptions, CrawleeLogger, IConcurrencySystem } from '@crawlee/core';
+import type { CrawleeLogger } from '@crawlee/core';
+import type { AutoscaledPoolOptions } from './autoscaling/autoscaled_pool.js';
+import type { IConcurrencySystem } from './autoscaling/concurrency_system.js';
 /** Everything a run needs to exist. Whatever a crawler resolves per run is resolved before the run is built. */
 export interface CrawlerRunSetup {
     log: CrawleeLogger;

package/internals/crawler-run.js

@@ -1,4 +1,4 @@
-import { AutoscaledPool } from '@crawlee/core';
+import { AutoscaledPool } from './autoscaling/autoscaled_pool.js';
 /**
  * Everything that belongs to a single {@link BasicCrawler.run|`crawler.run()`} — the task loop dispatching its
  * requests, the concurrency governor those requests are booked against, and the bookkeeping that must not survive

package/internals/crawlers/context_pipeline.d.ts

@@ -0,0 +1,70 @@
+import type { Awaitable } from '@crawlee/types';
+/**
+ * Represents a middleware step in the context pipeline.
+ *
+ * @template TCrawlingContext - The input context type for this middleware
+ * @template TCrawlingContextExtension - The enhanced output context type
+ */
+export interface ContextMiddleware<TCrawlingContext, TCrawlingContextExtension> {
+    /** The main middleware function that enhances the context */
+    action: (context: TCrawlingContext) => Awaitable<TCrawlingContextExtension>;
+    /** Optional cleanup function called after the consumer finishes or fails */
+    cleanup?: (context: TCrawlingContext & TCrawlingContextExtension, error?: unknown) => Awaitable<void>;
+}
+/**
+ * Encapsulates the logic of gradually enhancing the crawling context with additional information and utilities.
+ *
+ * The enhancement is done by a chain of middlewares that are added to the pipeline after its creation.
+ * This class provides a type-safe way to build a pipeline of context transformations where each step
+ * can enhance the context with additional properties or utilities.
+ *
+ * @template TContextBase - The base context type that serves as the starting point
+ * @template TCrawlingContext - The final context type after all middleware transformations
+ */
+export declare abstract class ContextPipeline<TContextBase, TCrawlingContext extends TContextBase> {
+    /**
+     * Creates a new empty context pipeline.
+     *
+     * @template TContextBase - The base context type for the pipeline
+     * @returns A new ContextPipeline instance with no transformations
+     */
+    static create<TContextBase>(): ContextPipeline<TContextBase, TContextBase>;
+    /**
+     * Adds a middleware to the pipeline, creating a new pipeline instance.
+     *
+     * This method provides a fluent interface for building context transformation pipelines.
+     * Each middleware can enhance the context with additional properties or utilities.
+     *
+     * @template TCrawlingContextExtension - The enhanced context type produced by this middleware
+     * @param middleware - The middleware to add to the pipeline
+     * @returns A new ContextPipeline instance with the added middleware
+     */
+    abstract compose<TCrawlingContextExtension>(middleware: ContextMiddleware<TCrawlingContext, TCrawlingContextExtension>): ContextPipeline<TContextBase, TCrawlingContext & TCrawlingContextExtension>;
+    /**
+     * Chains another pipeline onto this one. The other pipeline's base context must match
+     * this pipeline's output context. Returns a new pipeline that runs this pipeline's
+     * middlewares first, then the other pipeline's middlewares.
+     *
+     * @template TFinalContext - The final context type after the chained pipeline's transformations
+     * @param other - The pipeline to append after this one
+     * @returns A new ContextPipeline combining both pipelines' middlewares
+     */
+    abstract chain<TFinalContext extends TCrawlingContext>(other: ContextPipeline<TCrawlingContext, TFinalContext>): ContextPipeline<TContextBase, TFinalContext>;
+    /**
+     * Executes the middleware pipeline and passes the final context to a consumer function.
+     *
+     * This method runs the crawling context through the entire middleware chain, enhancing it
+     * at each step, and then passes the final enhanced context to the provided consumer function.
+     * Proper cleanup is performed even if exceptions occur during processing.
+     *
+     * @param crawlingContext - The initial context to process through the pipeline
+     * @param finalContextConsumer - The function that will receive the final enhanced context
+     *
+     * @throws {ContextPipelineInitializationError} When a middleware fails during initialization
+     * @throws {ContextPipelineInterruptedError} When the pipeline is intentionally interrupted during initialization
+     * @throws {RequestHandlerError} When the final context consumer throws an exception
+     * @throws {ContextPipelineCleanupError} When cleanup operations fail
+     * @throws {SessionError} Session errors are re-thrown as-is for special handling
+     */
+    abstract call(crawlingContext: TContextBase, finalContextConsumer: (finalContext: TCrawlingContext) => Awaitable<unknown>): Promise<void>;
+}

package/internals/crawlers/context_pipeline.js

@@ -0,0 +1,122 @@
+import { serviceLocator, SessionError } from '@crawlee/core';
+import { ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, RequestHandlerError, } from '../errors.js';
+/**
+ * Encapsulates the logic of gradually enhancing the crawling context with additional information and utilities.
+ *
+ * The enhancement is done by a chain of middlewares that are added to the pipeline after its creation.
+ * This class provides a type-safe way to build a pipeline of context transformations where each step
+ * can enhance the context with additional properties or utilities.
+ *
+ * @template TContextBase - The base context type that serves as the starting point
+ * @template TCrawlingContext - The final context type after all middleware transformations
+ */
+export class ContextPipeline {
+    /**
+     * Creates a new empty context pipeline.
+     *
+     * @template TContextBase - The base context type for the pipeline
+     * @returns A new ContextPipeline instance with no transformations
+     */
+    static create() {
+        return new ContextPipelineImpl({ action: async (context) => context });
+    }
+}
+/**
+ * Implementation of the `ContextPipeline` logic. This hides implementation details such as the `middleware` and `parent`
+ * properties from the `ContextPipeline` interface, making type checking more reliable.
+ */
+class ContextPipelineImpl extends ContextPipeline {
+    #middleware;
+    #parent;
+    constructor(middleware, parent) {
+        super();
+        this.#middleware = middleware;
+        this.#parent = parent;
+    }
+    /**
+     * @inheritdoc
+     */
+    compose(middleware) {
+        return new ContextPipelineImpl(middleware, this);
+    }
+    chain(other) {
+        const otherMiddlewares = Array.from(other.middlewareChain()).reverse();
+        let result = this;
+        for (const middleware of otherMiddlewares) {
+            result = result.compose(middleware);
+        }
+        return result;
+    }
+    *middlewareChain() {
+        let step = this;
+        while (step !== undefined) {
+            yield step.#middleware;
+            step = step.#parent;
+        }
+    }
+    /**
+     * @inheritdoc
+     */
+    async call(crawlingContext, finalContextConsumer) {
+        const middlewares = Array.from(this.middlewareChain()).reverse();
+        const cleanupStack = [];
+        let consumerException;
+        try {
+            for (const { action, cleanup } of middlewares) {
+                try {
+                    const contextExtension = await action(crawlingContext);
+                    const extensionNames = [
+                        ...Object.getOwnPropertyNames(contextExtension),
+                        ...Object.getOwnPropertySymbols(contextExtension),
+                    ];
+                    for (const key of extensionNames) {
+                        try {
+                            if (Object.getOwnPropertyDescriptor(crawlingContext, key)?.configurable !== false) {
+                                Object.defineProperty(crawlingContext, key, Object.getOwnPropertyDescriptor(contextExtension, key));
+                            }
+                        }
+                        catch (error) {
+                            serviceLocator
+                                .getLogger()
+                                .debug(`Context pipeline failed to define property ${key.toString()}:`, error);
+                        }
+                    }
+                    if (cleanup) {
+                        cleanupStack.push(cleanup);
+                    }
+                }
+                catch (exception) {
+                    if (exception instanceof SessionError) {
+                        throw exception; // Session errors are re-thrown as-is
+                    }
+                    if (exception instanceof ContextPipelineInterruptedError) {
+                        throw exception;
+                    }
+                    throw new ContextPipelineInitializationError(exception);
+                }
+            }
+            try {
+                await finalContextConsumer(crawlingContext);
+            }
+            catch (exception) {
+                if (exception instanceof SessionError) {
+                    consumerException = exception;
+                    throw exception; // Session errors are re-thrown as-is
+                }
+                consumerException = exception;
+                throw new RequestHandlerError(exception);
+            }
+        }
+        finally {
+            try {
+                for (const cleanup of cleanupStack.reverse()) {
+                    await cleanup(crawlingContext, consumerException);
+                }
+            }
+            catch (exception) {
+                // eslint-disable-next-line no-unsafe-finally
+                throw new ContextPipelineCleanupError(exception);
+            }
+        }
+    }
+}

package/internals/crawlers/crawler_commons.d.ts

@@ -0,0 +1,193 @@
+import type { Awaitable, Dictionary, HttpRequestOptions, ISession, ProxyInfo, SendRequestOptions } from '@crawlee/types';
+import type { ReadonlyDeep } from 'type-fest';
+import type { AddRequestsBatchedResult, CrawleeLogger, Dataset, KeyValueStore, Request, RequestOptions, Source, StorageIdentifier } from '@crawlee/core';
+import type { EnqueueUrlsOptions } from '../enqueue_links/index.js';
+/** @internal */
+export type IsAny<T> = 0 extends 1 & T ? true : false;
+/**
+ * A request input (URL string, request-options object, or {@link Request}) whose `userData` is typed
+ * according to its `label`, based on a router's route map.
+ *
+ * When the route map is open (the default `Record<string, ...>`), this is just the regular loose
+ * {@link Source} input. When the map declares concrete labels, providing a `label` requires the matching
+ * `userData` shape and rejects labels not present in the map; unlabeled requests keep loose `userData`.
+ */
+export type LabeledSource<Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes ? string | Source : string | Request | ({
+    requestsFromUrl?: string;
+    regex?: RegExp;
+} & ({
+    [Label in keyof Routes & string]: Omit<Partial<RequestOptions<Routes[Label]>>, 'label'> & {
+        label: Label;
+    };
+}[keyof Routes & string] | (Omit<Partial<RequestOptions>, 'label'> & {
+    label?: undefined;
+})));
+/**
+ * The iterable/array of {@link LabeledSource} inputs accepted by the label-aware `addRequests`/`run`
+ * methods of a crawler bound to a typed router.
+ */
+export type TypedRequestsLike<Routes extends Record<keyof Routes, Dictionary>> = AsyncIterable<LabeledSource<Routes>> | Iterable<LabeledSource<Routes>> | LabeledSource<Routes>[];
+/**
+ * The label-aware `addRequests` method signature exposed on a request handler's context when the crawler is
+ * bound to a typed router. Mirrors {@link RestrictedCrawlingContext.addRequests} with typed sources.
+ */
+export type TypedContextAddRequests<Routes extends Record<keyof Routes, Dictionary>> = (requestsLike: ReadonlyDeep<LabeledSource<Routes>[]>, options?: ReadonlyDeep<EnqueueUrlsOptions>) => Promise<AddRequestsBatchedResult>;
+/**
+ * An `enqueueLinks`-options object with its `label`/`userData` retyped according to a router's route map: a
+ * declared `label` requires the matching `userData` shape (unknown labels are rejected), while unlabeled
+ * calls keep loose `userData`. Returns the options unchanged when the route map is open (the default).
+ */
+type TypedEnqueueLinksOptions<Options, Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes ? Options : Omit<Options, 'label' | 'userData'> & ({
+    [Label in keyof Routes & string]: {
+        label: Label;
+        userData?: Routes[Label];
+    };
+}[keyof Routes & string] | {
+    label?: undefined;
+    userData?: Dictionary;
+});
+/**
+ * Transforms a context's existing `enqueueLinks` method so that the `label`/`userData` in its options follow
+ * the router's route map, while preserving everything else about the signature (argument optionality and
+ * return type, which differ between crawler types).
+ */
+export type TypedContextEnqueueLinks<EnqueueLinks, Routes extends Record<keyof Routes, Dictionary>> = EnqueueLinks extends (options?: infer Options) => infer Result ? (options?: TypedEnqueueLinksOptions<Options, Routes>) => Result : EnqueueLinks extends (options: infer Options) => infer Result ? (options: TypedEnqueueLinksOptions<Options, Routes>) => Result : EnqueueLinks;
+export type WithRequired<T, K extends keyof T> = T & {
+    [P in K]-?: T[P];
+};
+export type LoadedRequest<R extends Request> = WithRequired<R, 'id' | 'loadedUrl'>;
+/** @internal */
+export type LoadedContext<Context extends RestrictedCrawlingContext> = IsAny<Context> extends true ? Context : {
+    request: LoadedRequest<Context['request']>;
+} & Omit<Context, 'request'>;
+export interface RestrictedCrawlingContext<UserData extends Dictionary = Dictionary> {
+    id: string;
+    session: ISession;
+    /**
+     * An object with information about currently used proxy by the crawler
+     * and configured by the {@link ProxyConfiguration} class.
+     */
+    proxyInfo?: ProxyInfo;
+    /**
+     * The original {@link Request} object.
+     */
+    request: Request<UserData>;
+    /**
+     * This function allows you to push data to a {@link Dataset} specified by name, or the one currently used by the crawler.
+     *
+     * Shortcut for `crawler.pushData()`.
+     *
+     * @param [data] Data to be pushed to the default dataset.
+     */
+    pushData(data: ReadonlyDeep<Parameters<Dataset['pushData']>[0]>, datasetIdentifier?: string | StorageIdentifier): Promise<void>;
+    /**
+     * Add requests directly to the request queue currently used by the crawler.
+     *
+     * Optionally, the function allows you to filter the target URLs using an array of glob or regexp patterns,
+     * the same way {@link CrawlingContext.enqueueLinks|`enqueueLinks`} does for extracted links.
+     *
+     * @param requests The requests to add
+     * @param options Options for the request queue
+     */
+    addRequests: (requestsLike: ReadonlyDeep<(string | Source)[]>, options?: ReadonlyDeep<EnqueueUrlsOptions>) => Promise<AddRequestsBatchedResult>;
+    /**
+     * Returns the state - a piece of mutable persistent data shared across all the request handler runs.
+     */
+    useState: <State extends Dictionary = Dictionary>(defaultValue?: State) => Promise<State>;
+    /**
+     * Get a key-value store with given name or id, or the default one for the crawler.
+     */
+    getKeyValueStore: (identifier?: string | StorageIdentifier) => Promise<Pick<KeyValueStore, 'id' | 'name' | 'getValue' | 'getAutoSavedValue' | 'setValue' | 'getPublicUrl'>>;
+    /**
+     * A preconfigured logger for the request handler.
+     */
+    log: CrawleeLogger;
+}
+export interface CrawlingContext<UserData extends Dictionary = Dictionary> extends RestrictedCrawlingContext<UserData> {
+    /**
+     * Fires HTTP request via the internal HTTP client, allowing to override the request options on the fly.
+     *
+     * This is handy when you work with a browser crawler but want to execute some requests outside it (e.g. API requests).
+     * Check the [Skipping navigations for certain requests](https://crawlee.dev/js/docs/examples/skip-navigation) example for
+     * more detailed explanation of how to do that.
+     *
+     * ```ts
+     * async requestHandler({ sendRequest }) {
+     *     const { body } = await sendRequest({
+     *         // override headers only
+     *         headers: { ... },
+     *     });
+     * },
+     * ```
+     */
+    sendRequest: (requestOverrides?: Partial<HttpRequestOptions>, optionsOverrides?: SendRequestOptions) => Promise<Response>;
+    /**
+     * Register a function to be called at the very end of the request handling process. This is useful for resources that should be accessible to error handlers, for instance.
+     *
+     * The callback runs *outside* the request's storage transaction, so storage writes made here are
+     * applied immediately and are **not** rolled back when the request fails. In
+     * {@link AdaptivePlaywrightCrawler} it also runs once per request handler attempt, so a write
+     * here can land more than once for a single request. Push results from the request handler itself.
+     */
+    registerDeferredCleanup(cleanup: () => Promise<unknown>): void;
+    /**
+     * Registers `callback` to run once the request's storage writes have been committed, or once
+     * committing them has failed - the point where a write made in the handler can finally be reacted
+     * to where it was made. Two things belong here: side effects that must agree with what actually
+     * landed (result counters, progress reporting), and handling of a write that did not land at all.
+     *
+     * ```ts
+     * async requestHandler({ pushData, afterStorageCommit, useState }) {
+     *     const state = await useState({ itemCount: 0 });
+     *
+     *     await pushData(item);
+     *     afterStorageCommit((error) => {
+     *         if (error) {
+     *             throw new NonRetryableError('The page is too big to store', { cause: error });
+     *         }
+     *         state.itemCount++;
+     *     });
+     * }
+     * ```
+     *
+     * `useState()` is deliberately not transactional, which is what makes the counter case necessary:
+     * incrementing in the handler would keep the increment for a request that pushes an item and then
+     * fails, while the item itself is rolled back.
+     *
+     * Callbacks run in registration order, after the commit and before the request is marked as handled.
+     * The first one to throw propagates and the rest do not run; on a failed commit its error replaces
+     * the commit error, so a callback can decide what the request fails with. After a *successful*
+     * commit, a callback that throws fails the request **without a retry** whatever it throws - the
+     * writes are already durable and retrying would duplicate them, so anything that is not already a
+     * {@link NonRetryableError} is wrapped in an {@link AfterCommitError}.
+     *
+     * Callbacks are *not* run when the request handler itself fails - nothing was written, and
+     * `errorHandler` / `failedRequestHandler` cover that.
+     *
+     * Throws when storage is not transactional (`transactionalStorage: false`) - writes are then applied
+     * as they are made and throw at the call site on their own. In {@link AdaptivePlaywrightCrawler}
+     * the callback belongs to the current request handler attempt, so it runs only for the attempt whose
+     * writes are the ones being committed.
+     */
+    afterStorageCommit(callback: (error?: Error) => Awaitable<void>): void;
+    /**
+     * Gives the current request `secs` more seconds to finish, for when how long it needs is only apparent
+     * once it is already running - a listing page that turns out to have far more to scroll through than
+     * usual, say. Prefer `requestHandlerTimeoutSecs`, or a per-route override via
+     * {@link Router.addHandler|`router.addHandler`}, whenever the time needed is known up front.
+     *
+     * ```ts
+     * router.addHandler('LIST', async ({ extendTimeout, page }) => {
+     *     const pageCount = await countPages(page);
+     *     extendTimeout(pageCount * 10);
+     *     await scrapeAllPages(page);
+     * });
+     * ```
+     *
+     * Extends the request handler's own timeout and the crawler's internal one together, so the extension
+     * is not immediately undone by the latter. Calling it from a handler that has already timed out does
+     * nothing.
+     */
+    extendTimeout(secs: number): void;
+}
+export {};

package/internals/crawlers/crawler_commons.js

@@ -0,0 +1 @@
+export {};

package/internals/crawlers/error_snapshotter.d.ts

@@ -0,0 +1,57 @@
+import type { KeyValueStore } from '@crawlee/core';
+import type { CrawlingContext } from '../crawlers/crawler_commons.js';
+import type { ErrnoException } from './error_tracker.js';
+import type { SnapshottableProperties } from './internals/types.js';
+interface BrowserCrawlingContext {
+    saveSnapshot: (options: {
+        key: string;
+    }) => Promise<void>;
+}
+export interface SnapshotResult {
+    screenshotFileName?: string;
+    htmlFileName?: string;
+}
+interface ErrorSnapshot {
+    screenshotFileName?: string;
+    screenshotFileUrl?: string;
+    htmlFileName?: string;
+    htmlFileUrl?: string;
+}
+/**
+ * ErrorSnapshotter class is used to capture a screenshot of the page and a snapshot of the HTML when an error occurs during web crawling.
+ *
+ * This functionality is opt-in, and can be enabled via the crawler options:
+ *
+ * ```ts
+ * const crawler = new BasicCrawler({
+ *   // ...
+ *   statistics: new Statistics({ saveErrorSnapshots: true }),
+ * });
+ * ```
+ */
+export declare class ErrorSnapshotter {
+    static readonly MAX_ERROR_CHARACTERS = 30;
+    static readonly MAX_HASH_LENGTH = 30;
+    static readonly MAX_FILENAME_LENGTH = 250;
+    static readonly BASE_MESSAGE = "An error occurred";
+    static readonly SNAPSHOT_PREFIX = "ERROR_SNAPSHOT";
+    /**
+     * Capture a snapshot of the error context.
+     */
+    captureSnapshot(error: ErrnoException, context: CrawlingContext & SnapshottableProperties): Promise<ErrorSnapshot>;
+    /**
+     * Captures a snapshot of the current page using the context.saveSnapshot function.
+     * This function is applicable for browser contexts only.
+     * Returns an object containing the filenames of the screenshot and HTML file.
+     */
+    contextCaptureSnapshot(context: BrowserCrawlingContext, fileName: string): Promise<SnapshotResult | undefined>;
+    /**
+     * Save the HTML snapshot of the page, and return the key it was stored under.
+     */
+    saveHTMLSnapshot(html: string, keyValueStore: Pick<KeyValueStore, 'setValue'>, fileName: string): Promise<string | undefined>;
+    /**
+     * Generate a unique fileName for each error snapshot.
+     */
+    generateFilename(error: ErrnoException): string;
+}
+export {};