@crawlee/core 4.0.0-beta.8 → 4.0.0-beta.80

package/crawlers/context_pipeline.js

@@ -0,0 +1,122 @@
+import { ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, RequestHandlerError, SessionError, } from '../errors.js';
+import { serviceLocator } from '../service_locator.js';
+/**
+ * Encapsulates the logic of gradually enhancing the crawling context with additional information and utilities.
+ *
+ * The enhancement is done by a chain of middlewares that are added to the pipeline after its creation.
+ * This class provides a type-safe way to build a pipeline of context transformations where each step
+ * can enhance the context with additional properties or utilities.
+ *
+ * @template TContextBase - The base context type that serves as the starting point
+ * @template TCrawlingContext - The final context type after all middleware transformations
+ */
+export class ContextPipeline {
+    /**
+     * Creates a new empty context pipeline.
+     *
+     * @template TContextBase - The base context type for the pipeline
+     * @returns A new ContextPipeline instance with no transformations
+     */
+    static create() {
+        return new ContextPipelineImpl({ action: async (context) => context });
+    }
+}
+/**
+ * Implementation of the `ContextPipeline` logic. This hides implementation details such as the `middleware` and `parent`
+ * properties from the `ContextPipeline` interface, making type checking more reliable.
+ */
+class ContextPipelineImpl extends ContextPipeline {
+    middleware;
+    parent;
+    constructor(middleware, parent) {
+        super();
+        this.middleware = middleware;
+        this.parent = parent;
+    }
+    /**
+     * @inheritdoc
+     */
+    compose(middleware) {
+        return new ContextPipelineImpl(middleware, this);
+    }
+    chain(other) {
+        const otherMiddlewares = Array.from(other.middlewareChain()).reverse();
+        let result = this;
+        for (const middleware of otherMiddlewares) {
+            result = result.compose(middleware);
+        }
+        return result;
+    }
+    *middlewareChain() {
+        let step = this;
+        while (step !== undefined) {
+            yield step.middleware;
+            step = step.parent;
+        }
+    }
+    /**
+     * @inheritdoc
+     */
+    async call(crawlingContext, finalContextConsumer) {
+        const middlewares = Array.from(this.middlewareChain()).reverse();
+        const cleanupStack = [];
+        let consumerException;
+        try {
+            for (const { action, cleanup } of middlewares) {
+                try {
+                    const contextExtension = await action(crawlingContext);
+                    const extensionNames = [
+                        ...Object.getOwnPropertyNames(contextExtension),
+                        ...Object.getOwnPropertySymbols(contextExtension),
+                    ];
+                    for (const key of extensionNames) {
+                        try {
+                            if (Object.getOwnPropertyDescriptor(crawlingContext, key)?.configurable !== false) {
+                                Object.defineProperty(crawlingContext, key, Object.getOwnPropertyDescriptor(contextExtension, key));
+                            }
+                        }
+                        catch (error) {
+                            serviceLocator
+                                .getLogger()
+                                .debug(`Context pipeline failed to define property ${key.toString()}:`, error);
+                        }
+                    }
+                    if (cleanup) {
+                        cleanupStack.push(cleanup);
+                    }
+                }
+                catch (exception) {
+                    if (exception instanceof SessionError) {
+                        throw exception; // Session errors are re-thrown as-is
+                    }
+                    if (exception instanceof ContextPipelineInterruptedError) {
+                        throw exception;
+                    }
+                    throw new ContextPipelineInitializationError(exception);
+                }
+            }
+            try {
+                await finalContextConsumer(crawlingContext);
+            }
+            catch (exception) {
+                if (exception instanceof SessionError) {
+                    consumerException = exception;
+                    throw exception; // Session errors are re-thrown as-is
+                }
+                consumerException = exception;
+                throw new RequestHandlerError(exception);
+            }
+        }
+        finally {
+            try {
+                for (const cleanup of cleanupStack.reverse()) {
+                    await cleanup(crawlingContext, consumerException);
+                }
+            }
+            catch (exception) {
+                // eslint-disable-next-line no-unsafe-finally
+                throw new ContextPipelineCleanupError(exception);
+            }
+        }
+    }
+}

package/crawlers/crawler_commons.d.ts

@@ -1,16 +1,13 @@
-import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
-// @ts-ignore optional peer dependency or compatibility with es2022
-import type { OptionsInit, Response as GotResponse } from 'got-scraping';
-import type { ReadonlyDeep } from 'type-fest';
+import type { Dictionary, HttpRequestOptions, ISession, ProxyInfo, SendRequestOptions } from '@crawlee/types';
+import type { ReadonlyDeep, SetRequired } from 'type-fest';
 import type { Configuration } from '../configuration.js';
 import type { EnqueueLinksOptions } from '../enqueue_links/enqueue_links.js';
-import type { Log } from '../log.js';
-import type { ProxyInfo } from '../proxy_configuration.js';
+import type { CrawleeLogger } from '../log.js';
 import type { Request, Source } from '../request.js';
-import type { Session } from '../session_pool/session.js';
 import type { Dataset } from '../storages/dataset.js';
 import { KeyValueStore, type RecordOptions } from '../storages/key_value_store.js';
-import type { RequestQueueOperationOptions } from '../storages/request_provider.js';
+import type { RequestQueueOperationOptions } from '../storages/request_queue.js';
+import type { StorageIdentifier } from '../storages/storage_instance_manager.js';
 /** @internal */
 export type IsAny<T> = 0 extends 1 & T ? true : false;
 /** @internal */
@@ -22,9 +19,9 @@ export type LoadedRequest<R extends Request> = WithRequired<R, 'id' | 'loadedUrl
 export type LoadedContext<Context extends RestrictedCrawlingContext> = IsAny<Context> extends true ? Context : {
     request: LoadedRequest<Context['request']>;
 } & Omit<Context, 'request'>;
-export interface RestrictedCrawlingContext<UserData extends Dictionary = Dictionary> extends Record<string & {}, unknown> {
+export interface RestrictedCrawlingContext<UserData extends Dictionary = Dictionary> {
     id: string;
-    session?: Session;
+    session: ISession;
     /**
      * An object with information about currently used proxy by the crawler
      * and configured by the {@link ProxyConfiguration} class.
@@ -41,7 +38,7 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
      *
      * @param [data] Data to be pushed to the default dataset.
      */
-    pushData(data: ReadonlyDeep<Parameters<Dataset['pushData']>[0]>, datasetIdOrName?: string): Promise<void>;
+    pushData(data: ReadonlyDeep<Parameters<Dataset['pushData']>[0]>, datasetIdentifier?: string | StorageIdentifier): Promise<void>;
     /**
      * This function automatically finds and enqueues links from the current page, adding them to the {@link RequestQueue}
      * currently used by the crawler.
@@ -66,7 +63,7 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
      *
      * @param [options] All `enqueueLinks()` parameters are passed via an options object.
      */
-    enqueueLinks: (options?: ReadonlyDeep<Omit<EnqueueLinksOptions, 'requestQueue'>>) => Promise<unknown>;
+    enqueueLinks: (options: ReadonlyDeep<Omit<SetRequired<EnqueueLinksOptions, 'urls'>, 'requestManager' | 'robotsTxtFile'>>) => Promise<unknown>;
     /**
      * Add requests directly to the request queue.
      *
@@ -81,14 +78,13 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
     /**
      * Get a key-value store with given name or id, or the default one for the crawler.
      */
-    getKeyValueStore: (idOrName?: string) => Promise<Pick<KeyValueStore, 'id' | 'name' | 'getValue' | 'getAutoSavedValue' | 'setValue' | 'getPublicUrl'>>;
+    getKeyValueStore: (identifier?: string | StorageIdentifier) => Promise<Pick<KeyValueStore, 'id' | 'name' | 'getValue' | 'getAutoSavedValue' | 'setValue' | 'getPublicUrl'>>;
     /**
      * A preconfigured logger for the request handler.
      */
-    log: Log;
+    log: CrawleeLogger;
 }
-export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary = Dictionary> extends RestrictedCrawlingContext<UserData> {
-    crawler: Crawler;
+export interface CrawlingContext<UserData extends Dictionary = Dictionary> extends RestrictedCrawlingContext<UserData> {
     /**
      * This function automatically finds and enqueues links from the current page, adding them to the {@link RequestQueue}
      * currently used by the crawler.
@@ -114,14 +110,9 @@ export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary
      * @param [options] All `enqueueLinks()` parameters are passed via an options object.
      * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
      */
-    enqueueLinks(options?: ReadonlyDeep<Omit<EnqueueLinksOptions, 'requestQueue'>> & Pick<EnqueueLinksOptions, 'requestQueue'>): Promise<BatchAddRequestsResult>;
+    enqueueLinks(options: ReadonlyDeep<Omit<SetRequired<EnqueueLinksOptions, 'urls'>, 'requestManager' | 'robotsTxtFile'>> & Pick<EnqueueLinksOptions, 'requestManager' | 'robotsTxtFile'>): Promise<unknown>;
     /**
-     * Get a key-value store with given name or id, or the default one for the crawler.
-     */
-    getKeyValueStore: (idOrName?: string) => Promise<KeyValueStore>;
-    /**
-     * Fires HTTP request via [`got-scraping`](https://crawlee.dev/js/docs/guides/got-scraping), allowing to override the request
-     * options on the fly.
+     * Fires HTTP request via the internal HTTP client, allowing to override the request options on the fly.
      *
      * This is handy when you work with a browser crawler but want to execute some requests outside it (e.g. API requests).
      * Check the [Skipping navigations for certain requests](https://crawlee.dev/js/docs/examples/skip-navigation) example for
@@ -136,7 +127,11 @@ export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary
      * },
      * ```
      */
-    sendRequest<Response = string>(overrideOptions?: Partial<OptionsInit>): Promise<GotResponse<Response>>;
+    sendRequest: (requestOverrides?: Partial<HttpRequestOptions>, optionsOverrides?: SendRequestOptions) => Promise<Response>;
+    /**
+     * Register a function to be called at the very end of the request handling process. This is useful for resources that should be accessible to error handlers, for instance.
+     */
+    registerDeferredCleanup(cleanup: () => Promise<unknown>): void;
 }
 /**
  * A partial implementation of {@link RestrictedCrawlingContext} that stores parameters of calls to context methods for later inspection.
@@ -149,7 +144,6 @@ export declare class RequestHandlerResult {
     private _keyValueStoreChanges;
     private pushDataCalls;
     private addRequestsCalls;
-    private enqueueLinksCalls;
     constructor(config: Configuration, crawleeStateKey: string);
     /**
      * A record of calls to {@link RestrictedCrawlingContext.pushData}, {@link RestrictedCrawlingContext.addRequests}, {@link RestrictedCrawlingContext.enqueueLinks} made by a request handler.
@@ -157,7 +151,6 @@ export declare class RequestHandlerResult {
     get calls(): ReadonlyDeep<{
         pushData: Parameters<RestrictedCrawlingContext['pushData']>[];
         addRequests: Parameters<RestrictedCrawlingContext['addRequests']>[];
-        enqueueLinks: Parameters<RestrictedCrawlingContext['enqueueLinks']>[];
     }>;
     /**
      * A record of changes made to key-value stores by a request handler.
@@ -171,7 +164,7 @@ export declare class RequestHandlerResult {
      */
     get datasetItems(): ReadonlyDeep<{
         item: Dictionary;
-        datasetIdOrName?: string;
+        datasetIdentifier?: string | StorageIdentifier;
     }[]>;
     /**
      * URLs enqueued to the request queue by a request handler, either via {@link RestrictedCrawlingContext.addRequests} or {@link RestrictedCrawlingContext.enqueueLinks}
@@ -188,12 +181,9 @@ export declare class RequestHandlerResult {
         label?: string;
     }[]>;
     pushData: RestrictedCrawlingContext['pushData'];
-    enqueueLinks: RestrictedCrawlingContext['enqueueLinks'];
     addRequests: RestrictedCrawlingContext['addRequests'];
     useState: RestrictedCrawlingContext['useState'];
     getKeyValueStore: RestrictedCrawlingContext['getKeyValueStore'];
-    private idOrDefault;
     private getKeyValueStoreChangedValue;
     private setKeyValueStoreChangedValue;
 }
-//# sourceMappingURL=crawler_commons.d.ts.map

package/crawlers/crawler_commons.js

@@ -10,7 +10,6 @@ export class RequestHandlerResult {
     _keyValueStoreChanges = {};
     pushDataCalls = [];
     addRequestsCalls = [];
-    enqueueLinksCalls = [];
     constructor(config, crawleeStateKey) {
         this.config = config;
         this.crawleeStateKey = crawleeStateKey;
@@ -22,7 +21,6 @@ export class RequestHandlerResult {
         return {
             pushData: this.pushDataCalls,
             addRequests: this.addRequestsCalls,
-            enqueueLinks: this.enqueueLinksCalls,
         };
     }
     /**
@@ -35,16 +33,13 @@ export class RequestHandlerResult {
      * Items added to datasets by a request handler.
      */
     get datasetItems() {
-        return this.pushDataCalls.flatMap(([data, datasetIdOrName]) => (Array.isArray(data) ? data : [data]).map((item) => ({ item, datasetIdOrName })));
+        return this.pushDataCalls.flatMap(([data, datasetIdentifier]) => (Array.isArray(data) ? data : [data]).map((item) => ({ item, datasetIdentifier })));
     }
     /**
      * URLs enqueued to the request queue by a request handler, either via {@link RestrictedCrawlingContext.addRequests} or {@link RestrictedCrawlingContext.enqueueLinks}
      */
     get enqueuedUrls() {
         const result = [];
-        for (const [options] of this.enqueueLinksCalls) {
-            result.push(...(options?.urls?.map((url) => ({ url, label: options?.label })) ?? []));
-        }
         for (const [requests] of this.addRequestsCalls) {
             for (const request of requests) {
                 if (typeof request === 'object' &&
@@ -78,9 +73,6 @@ export class RequestHandlerResult {
     pushData = async (data, datasetIdOrName) => {
         this.pushDataCalls.push([data, datasetIdOrName]);
     };
-    enqueueLinks = async (options) => {
-        this.enqueueLinksCalls.push([options]);
-    };
     addRequests = async (requests, options = {}) => {
         this.addRequestsCalls.push([requests, options]);
     };
@@ -88,29 +80,28 @@ export class RequestHandlerResult {
         const store = await this.getKeyValueStore(undefined);
         return await store.getAutoSavedValue(this.crawleeStateKey, defaultValue);
     };
-    getKeyValueStore = async (idOrName) => {
-        const store = await KeyValueStore.open(idOrName, { config: this.config });
+    getKeyValueStore = async (identifier) => {
+        const store = await KeyValueStore.open(identifier, { config: this.config });
+        const storeId = store.id;
         return {
-            id: this.idOrDefault(idOrName),
-            name: idOrName,
-            getValue: async (key) => this.getKeyValueStoreChangedValue(idOrName, key) ?? (await store.getValue(key)),
+            id: storeId ?? this.config.defaultKeyValueStoreId,
+            name: store.name,
+            getValue: async (key) => this.getKeyValueStoreChangedValue(storeId, key) ?? (await store.getValue(key)),
             setValue: async (key, value, options) => {
-                this.setKeyValueStoreChangedValue(idOrName, key, value, options);
+                this.setKeyValueStoreChangedValue(storeId, key, value, options);
             },
             getAutoSavedValue: store.getAutoSavedValue.bind(store),
             getPublicUrl: store.getPublicUrl.bind(store),
         };
     };
-    idOrDefault = (idOrName) => idOrName ?? this.config.get('defaultKeyValueStoreId');
-    getKeyValueStoreChangedValue = (idOrName, key) => {
-        const id = this.idOrDefault(idOrName);
+    getKeyValueStoreChangedValue = (storeKey, key) => {
+        const id = storeKey ?? this.config.defaultKeyValueStoreId;
         this._keyValueStoreChanges[id] ??= {};
         return this.keyValueStoreChanges[id][key]?.changedValue ?? null;
     };
-    setKeyValueStoreChangedValue = (idOrName, key, changedValue, options) => {
-        const id = this.idOrDefault(idOrName);
+    setKeyValueStoreChangedValue = (storeKey, key, changedValue, options) => {
+        const id = storeKey ?? this.config.defaultKeyValueStoreId;
         this._keyValueStoreChanges[id] ??= {};
         this._keyValueStoreChanges[id][key] = { changedValue, options };
     };
 }
-//# sourceMappingURL=crawler_commons.js.map

package/crawlers/crawler_utils.d.ts

@@ -1,10 +1,9 @@
-import type { Session } from '../session_pool/session.js';
+import type { ISession } from '@crawlee/types';
 /**
  * Handles timeout request
  * @internal
  */
 export declare function handleRequestTimeout({ session, errorMessage }: {
-    session?: Session;
+    session?: ISession;
     errorMessage: string;
 }): void;
-//# sourceMappingURL=crawler_utils.d.ts.map

package/crawlers/crawler_utils.js

@@ -5,8 +5,7 @@ import { TimeoutError } from '@apify/timeout';
  */
 export function handleRequestTimeout({ session, errorMessage }) {
     session?.markBad();
-    const timeoutMillis = errorMessage.match(/(\d+)\s?ms/)?.[1]; // first capturing group
+    const timeoutMillis = /(\d+)\s?ms/.exec(errorMessage)?.[1]; // first capturing group
     const timeoutSecs = Number(timeoutMillis) / 1000;
     throw new TimeoutError(`Navigation timed out after ${timeoutSecs} seconds.`);
 }
-//# sourceMappingURL=crawler_utils.js.map

package/crawlers/error_snapshotter.d.ts

@@ -1,6 +1,7 @@
 import type { CrawlingContext } from '../crawlers/crawler_commons.js';
 import type { KeyValueStore } from '../storages/key_value_store.js';
 import type { ErrnoException } from './error_tracker.js';
+import type { SnapshottableProperties } from './internals/types.js';
 interface BrowserCrawlingContext {
     saveSnapshot: (options: {
         key: string;
@@ -39,7 +40,7 @@ export declare class ErrorSnapshotter {
     /**
      * Capture a snapshot of the error context.
      */
-    captureSnapshot(error: ErrnoException, context: CrawlingContext): Promise<ErrorSnapshot>;
+    captureSnapshot(error: ErrnoException, context: CrawlingContext & SnapshottableProperties): Promise<ErrorSnapshot>;
     /**
      * Captures a snapshot of the current page using the context.saveSnapshot function.
      * This function is applicable for browser contexts only.
@@ -49,11 +50,10 @@ export declare class ErrorSnapshotter {
     /**
      * Save the HTML snapshot of the page, and return the fileName with the extension.
      */
-    saveHTMLSnapshot(html: string, keyValueStore: KeyValueStore, fileName: string): Promise<string | undefined>;
+    saveHTMLSnapshot(html: string, keyValueStore: Pick<KeyValueStore, 'setValue'>, fileName: string): Promise<string | undefined>;
     /**
      * Generate a unique fileName for each error snapshot.
      */
     generateFilename(error: ErrnoException): string;
 }
 export {};
-//# sourceMappingURL=error_snapshotter.d.ts.map

package/crawlers/error_snapshotter.js

@@ -52,9 +52,9 @@ export class ErrorSnapshotter {
             }
             return {
                 screenshotFileName,
-                screenshotFileUrl: screenshotFileName && keyValueStore.getPublicUrl(screenshotFileName),
+                screenshotFileUrl: screenshotFileName && (await keyValueStore.getPublicUrl(screenshotFileName)),
                 htmlFileName,
-                htmlFileUrl: htmlFileName && keyValueStore.getPublicUrl(htmlFileName),
+                htmlFileUrl: htmlFileName && (await keyValueStore.getPublicUrl(htmlFileName)),
             };
         }
         catch {
@@ -115,4 +115,3 @@ export class ErrorSnapshotter {
         return fileName;
     }
 }
-//# sourceMappingURL=error_snapshotter.js.map

package/crawlers/error_tracker.d.ts

@@ -1,5 +1,6 @@
 import type { CrawlingContext } from '../crawlers/crawler_commons.js';
 import { ErrorSnapshotter } from './error_snapshotter.js';
+import type { SnapshottableProperties } from './internals/types.js';
 /**
  * Node.js Error interface
  */
@@ -48,7 +49,6 @@ export declare class ErrorTracker {
     addAsync(error: ErrnoException, context?: CrawlingContext): Promise<void>;
     getUniqueErrorCount(): number;
     getMostPopularErrors(count: number): [number, string[]][];
-    captureSnapshot(storage: Record<string, unknown>, error: ErrnoException, context: CrawlingContext): Promise<void>;
+    captureSnapshot(storage: Record<string, unknown>, error: ErrnoException, context: CrawlingContext & SnapshottableProperties): Promise<void>;
     reset(): void;
 }
-//# sourceMappingURL=error_tracker.d.ts.map

package/crawlers/error_tracker.js

@@ -306,4 +306,3 @@ export class ErrorTracker {
         }
     }
 }
-//# sourceMappingURL=error_tracker.js.map

package/crawlers/index.d.ts

@@ -1,7 +1,6 @@
+export * from './context_pipeline.js';
 export * from './crawler_commons.js';
-export * from './crawler_extension.js';
 export * from './crawler_utils.js';
 export * from './statistics.js';
 export * from './error_tracker.js';
 export * from './error_snapshotter.js';
-//# sourceMappingURL=index.d.ts.map

package/crawlers/index.js

@@ -1,7 +1,6 @@
+export * from './context_pipeline.js';
 export * from './crawler_commons.js';
-export * from './crawler_extension.js';
 export * from './crawler_utils.js';
 export * from './statistics.js';
 export * from './error_tracker.js';
 export * from './error_snapshotter.js';
-//# sourceMappingURL=index.js.map

package/crawlers/internals/types.d.ts

@@ -0,0 +1,7 @@
+export interface BrowserPage {
+    content: () => Promise<string>;
+}
+export interface SnapshottableProperties {
+    body?: unknown;
+    page?: BrowserPage;
+}

package/crawlers/internals/types.js

@@ -0,0 +1 @@
+export {};

package/crawlers/statistics.d.ts

@@ -1,5 +1,4 @@
-import type { Log } from '@apify/log';
-import { Configuration } from '../configuration.js';
+import type { CrawleeLogger } from '../log.js';
 import { KeyValueStore } from '../storages/key_value_store.js';
 import { ErrorTracker } from './error_tracker.js';
 /**
@@ -17,7 +16,7 @@ export interface PersistenceOptions {
  * statistics for requests.
  *
  * All statistic information is saved on key value store
- * under the key `SDK_CRAWLER_STATISTICS_*`, persists between
+ * under the key `CRAWLEE_CRAWLER_STATISTICS_*`, persists between
  * migrations and abort/resurrect
  *
  * @category Crawlers
@@ -35,7 +34,7 @@ export declare class Statistics {
     /**
      * Statistic instance id.
      */
-    readonly id: number;
+    readonly id: string;
     /**
      * Current statistic state used for doing calculations on {@link Statistics.calculate} calls
      */
@@ -44,10 +43,6 @@ export declare class Statistics {
      * Contains the current retries histogram. Index 0 means 0 retries, index 2, 2 retries, and so on
      */
     readonly requestRetryHistogram: number[];
-    /**
-     * Contains the associated Configuration instance
-     */
-    private readonly config;
     protected keyValueStore?: KeyValueStore;
     protected persistStateKey: string;
     private logIntervalMillis;
@@ -57,8 +52,9 @@ export declare class Statistics {
     private readonly log;
     private instanceStart;
     private logInterval;
-    private events;
+    private _events?;
     private persistenceOptions;
+    private get events();
     /**
      * @internal
      */
@@ -90,6 +86,12 @@ export declare class Statistics {
      * @ignore
      */
     failJob(id: number | string, retryCount: number): void;
+    /**
+     * Discards a started job without affecting the finished/failed counters, e.g. when a request
+     * turns out to be skipped (robots.txt, enqueue strategy) after `startJob` was already called for it.
+     * @ignore
+     */
+    discardJob(id: number | string): void;
     /**
      * Calculate the current statistics
      */
@@ -146,17 +148,12 @@ export interface StatisticsOptions {
      * Parent logger instance, the statistics will create a child logger from this.
      * @default crawler.log
      */
-    log?: Log;
+    log?: CrawleeLogger;
     /**
      * Key value store instance to persist the statistics.
      * If not provided, the default one will be used when capturing starts
      */
     keyValueStore?: KeyValueStore;
-    /**
-     * Configuration instance to use
-     * @default Configuration.getGlobalConfig()
-     */
-    config?: Configuration;
     /**
      * Control how and when to persist the statistics.
      */
@@ -166,13 +163,22 @@ export interface StatisticsOptions {
      * @default false
      */
     saveErrorSnapshots?: boolean;
+    /**
+     * A unique identifier for this statistics instance. This ID is used for persistence
+     * to the key value store, ensuring the same statistics can be loaded after script restarts.
+     *
+     * If not provided, an auto-incremented ID will be used for backward compatibility.
+     * This means statistics may not persist correctly across script restarts
+     * if crawler creation order changes.
+     */
+    id?: string;
 }
 /**
  * Format of the persisted stats
  */
 export interface StatisticPersistedState extends Omit<StatisticState, 'statsPersistedAt'> {
     requestRetryHistogram: number[];
-    statsId: number;
+    statsId: string;
     requestAvgFailedDurationMillis: number;
     requestAvgFinishedDurationMillis: number;
     requestTotalDurationMillis: number;
@@ -201,4 +207,3 @@ export interface StatisticState {
     retryErrors: Record<string, unknown>;
     requestsWithStatusCode: Record<string, number>;
 }
-//# sourceMappingURL=statistics.d.ts.map