@crawlee/core 3.0.3-beta.2 → 3.0.3-beta.20

package/autoscaling/autoscaled_pool.d.ts

@@ -20,7 +20,7 @@ export interface AutoscaledPoolOptions {
      * If it resolves to `true` then the pool's run finishes. Being called only
      * when there are no tasks being processed means that as long as `isTaskReadyFunction()`
      * keeps resolving to `true`, `isFinishedFunction()` will never be called.
-     * To abort a run, use the {@link AutoscaledPool.abort} method.
+     * To abort a run, use the {@apilink AutoscaledPool.abort} method.
      */
     isFinishedFunction?: () => Promise<boolean>;
     /**
@@ -84,12 +84,12 @@ export interface AutoscaledPoolOptions {
      */
     taskTimeoutSecs?: number;
     /**
-     * Options to be passed down to the {@link Snapshotter} constructor. This is useful for fine-tuning
+     * Options to be passed down to the {@apilink Snapshotter} constructor. This is useful for fine-tuning
      * the snapshot intervals and history.
      */
     snapshotterOptions?: SnapshotterOptions;
     /**
-     * Options to be passed down to the {@link SystemStatus} constructor. This is useful for fine-tuning
+     * Options to be passed down to the {@apilink SystemStatus} constructor. This is useful for fine-tuning
      * the system status reports. If a custom snapshotter is set in the options, it will be used
      * by the pool.
      */
@@ -106,22 +106,22 @@ export interface AutoscaledPoolOptions {
  * The pool only starts new tasks if there is enough free CPU and memory available
  * and the Javascript event loop is not blocked.
  *
- * The information about the CPU and memory usage is obtained by the {@link Snapshotter} class,
+ * The information about the CPU and memory usage is obtained by the {@apilink Snapshotter} class,
  * which makes regular snapshots of system resources that may be either local
  * or from the Apify cloud infrastructure in case the process is running on the Apify platform.
- * Meaningful data gathered from these snapshots is provided to `AutoscaledPool` by the {@link SystemStatus} class.
+ * Meaningful data gathered from these snapshots is provided to `AutoscaledPool` by the {@apilink SystemStatus} class.
  *
  * Before running the pool, you need to implement the following three functions:
- * {@link AutoscaledPoolOptions.runTaskFunction},
- * {@link AutoscaledPoolOptions.isTaskReadyFunction} and
- * {@link AutoscaledPoolOptions.isFinishedFunction}.
+ * {@apilink AutoscaledPoolOptions.runTaskFunction},
+ * {@apilink AutoscaledPoolOptions.isTaskReadyFunction} and
+ * {@apilink AutoscaledPoolOptions.isFinishedFunction}.
  *
- * The auto-scaled pool is started by calling the {@link AutoscaledPool.run} function.
- * The pool periodically queries the {@link AutoscaledPoolOptions.isTaskReadyFunction} function
+ * The auto-scaled pool is started by calling the {@apilink AutoscaledPool.run} function.
+ * The pool periodically queries the {@apilink AutoscaledPoolOptions.isTaskReadyFunction} function
  * for more tasks, managing optimal concurrency, until the function resolves to `false`. The pool then queries
- * the {@link AutoscaledPoolOptions.isFinishedFunction}. If it resolves to `true`, the run finishes after all running tasks complete.
+ * the {@apilink AutoscaledPoolOptions.isFinishedFunction}. If it resolves to `true`, the run finishes after all running tasks complete.
  * If it resolves to `false`, it assumes there will be more tasks available later and keeps periodically querying for tasks.
- * If any of the tasks throws then the {@link AutoscaledPool.run} function rejects the promise with an error.
+ * If any of the tasks throws then the {@apilink AutoscaledPool.run} function rejects the promise with an error.
  *
  * The pool evaluates whether it should start a new task every time one of the tasks finishes
  * and also in the interval set by the `options.maybeRunIntervalSecs` parameter.
@@ -221,7 +221,7 @@ export declare class AutoscaledPool {
     run(): Promise<void>;
     /**
      * Aborts the run of the auto-scaled pool and destroys it. The promise returned from
-     * the {@link AutoscaledPool.run} function will immediately resolve, no more new tasks
+     * the {@apilink AutoscaledPool.run} function will immediately resolve, no more new tasks
      * will be spawned and all running tasks will be left in their current state.
      *
      * Due to the nature of the tasks, auto-scaled pool cannot reliably guarantee abortion
@@ -233,19 +233,19 @@ export declare class AutoscaledPool {
     abort(): Promise<void>;
     /**
      * Prevents the auto-scaled pool from starting new tasks, but allows the running ones to finish
-     * (unlike abort, which terminates them). Used together with {@link AutoscaledPool.resume}
+     * (unlike abort, which terminates them). Used together with {@apilink AutoscaledPool.resume}
      *
      * The function's promise will resolve once all running tasks have completed and the pool
      * is effectively idle. If the `timeoutSecs` argument is provided, the promise will reject
      * with a timeout error after the `timeoutSecs` seconds.
      *
-     * The promise returned from the {@link AutoscaledPool.run} function will not resolve
+     * The promise returned from the {@apilink AutoscaledPool.run} function will not resolve
      * when `.pause()` is invoked (unlike abort, which resolves it).
      */
     pause(timeoutSecs?: number): Promise<void>;
     /**
      * Resumes the operation of the autoscaled-pool by allowing more tasks to be run.
-     * Used together with {@link AutoscaledPool.pause}
+     * Used together with {@apilink AutoscaledPool.pause}
      *
      * Tasks will automatically start running again in `options.maybeRunIntervalSecs`.
      */

package/autoscaling/autoscaled_pool.js

@@ -14,22 +14,22 @@ const system_status_1 = require("./system_status");
  * The pool only starts new tasks if there is enough free CPU and memory available
  * and the Javascript event loop is not blocked.
  *
- * The information about the CPU and memory usage is obtained by the {@link Snapshotter} class,
+ * The information about the CPU and memory usage is obtained by the {@apilink Snapshotter} class,
  * which makes regular snapshots of system resources that may be either local
  * or from the Apify cloud infrastructure in case the process is running on the Apify platform.
- * Meaningful data gathered from these snapshots is provided to `AutoscaledPool` by the {@link SystemStatus} class.
+ * Meaningful data gathered from these snapshots is provided to `AutoscaledPool` by the {@apilink SystemStatus} class.
  *
  * Before running the pool, you need to implement the following three functions:
- * {@link AutoscaledPoolOptions.runTaskFunction},
- * {@link AutoscaledPoolOptions.isTaskReadyFunction} and
- * {@link AutoscaledPoolOptions.isFinishedFunction}.
+ * {@apilink AutoscaledPoolOptions.runTaskFunction},
+ * {@apilink AutoscaledPoolOptions.isTaskReadyFunction} and
+ * {@apilink AutoscaledPoolOptions.isFinishedFunction}.
  *
- * The auto-scaled pool is started by calling the {@link AutoscaledPool.run} function.
- * The pool periodically queries the {@link AutoscaledPoolOptions.isTaskReadyFunction} function
+ * The auto-scaled pool is started by calling the {@apilink AutoscaledPool.run} function.
+ * The pool periodically queries the {@apilink AutoscaledPoolOptions.isTaskReadyFunction} function
  * for more tasks, managing optimal concurrency, until the function resolves to `false`. The pool then queries
- * the {@link AutoscaledPoolOptions.isFinishedFunction}. If it resolves to `true`, the run finishes after all running tasks complete.
+ * the {@apilink AutoscaledPoolOptions.isFinishedFunction}. If it resolves to `true`, the run finishes after all running tasks complete.
  * If it resolves to `false`, it assumes there will be more tasks available later and keeps periodically querying for tasks.
- * If any of the tasks throws then the {@link AutoscaledPool.run} function rejects the promise with an error.
+ * If any of the tasks throws then the {@apilink AutoscaledPool.run} function rejects the promise with an error.
  *
  * The pool evaluates whether it should start a new task every time one of the tasks finishes
  * and also in the interval set by the `options.maybeRunIntervalSecs` parameter.
@@ -378,7 +378,7 @@ class AutoscaledPool {
     }
     /**
      * Aborts the run of the auto-scaled pool and destroys it. The promise returned from
-     * the {@link AutoscaledPool.run} function will immediately resolve, no more new tasks
+     * the {@apilink AutoscaledPool.run} function will immediately resolve, no more new tasks
      * will be spawned and all running tasks will be left in their current state.
      *
      * Due to the nature of the tasks, auto-scaled pool cannot reliably guarantee abortion
@@ -396,13 +396,13 @@ class AutoscaledPool {
     }
     /**
      * Prevents the auto-scaled pool from starting new tasks, but allows the running ones to finish
-     * (unlike abort, which terminates them). Used together with {@link AutoscaledPool.resume}
+     * (unlike abort, which terminates them). Used together with {@apilink AutoscaledPool.resume}
      *
      * The function's promise will resolve once all running tasks have completed and the pool
      * is effectively idle. If the `timeoutSecs` argument is provided, the promise will reject
      * with a timeout error after the `timeoutSecs` seconds.
      *
-     * The promise returned from the {@link AutoscaledPool.run} function will not resolve
+     * The promise returned from the {@apilink AutoscaledPool.run} function will not resolve
      * when `.pause()` is invoked (unlike abort, which resolves it).
      */
     async pause(timeoutSecs) {
@@ -431,7 +431,7 @@ class AutoscaledPool {
     }
     /**
      * Resumes the operation of the autoscaled-pool by allowing more tasks to be run.
-     * Used together with {@link AutoscaledPool.pause}
+     * Used together with {@apilink AutoscaledPool.pause}
      *
      * Tasks will automatically start running again in `options.maybeRunIntervalSecs`.
      */

package/autoscaling/snapshotter.d.ts

@@ -75,7 +75,7 @@ interface ClientSnapshot {
  * Creates snapshots of system resources at given intervals and marks the resource
  * as either overloaded or not during the last interval. Keeps a history of the snapshots.
  * It tracks the following resources: Memory, EventLoop, API and CPU.
- * The class is used by the {@link AutoscaledPool} class.
+ * The class is used by the {@apilink AutoscaledPool} class.
  *
  * When running on the Apify platform, the CPU and memory statistics are provided by the platform,
  * as collected from the running Docker container. When running locally, `Snapshotter`

package/autoscaling/snapshotter.js

@@ -14,7 +14,7 @@ const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10000;
  * Creates snapshots of system resources at given intervals and marks the resource
  * as either overloaded or not during the last interval. Keeps a history of the snapshots.
  * It tracks the following resources: Memory, EventLoop, API and CPU.
- * The class is used by the {@link AutoscaledPool} class.
+ * The class is used by the {@apilink AutoscaledPool} class.
  *
  * When running on the Apify platform, the CPU and memory statistics are provided by the platform,
  * as collected from the running Docker container. When running locally, `Snapshotter`

package/autoscaling/system_status.d.ts

@@ -29,7 +29,7 @@ export interface SystemInfo {
 }
 export interface SystemStatusOptions {
     /**
-     * Defines max age of snapshots used in the {@link SystemStatus.getCurrentStatus} measurement.
+     * Defines max age of snapshots used in the {@apilink SystemStatus.getCurrentStatus} measurement.
      * @default 5
      */
     currentHistorySecs?: number;
@@ -82,25 +82,25 @@ export interface FinalStatistics {
     crawlerRuntimeMillis: number;
 }
 /**
- * Provides a simple interface to reading system status from a {@link Snapshotter} instance.
- * It only exposes two functions {@link SystemStatus.getCurrentStatus}
- * and {@link SystemStatus.getHistoricalStatus}.
+ * Provides a simple interface to reading system status from a {@apilink Snapshotter} instance.
+ * It only exposes two functions {@apilink SystemStatus.getCurrentStatus}
+ * and {@apilink SystemStatus.getHistoricalStatus}.
  * The system status is calculated using a weighted average of overloaded
  * messages in the snapshots, with the weights being the time intervals
  * between the snapshots. Each resource is calculated separately
  * and the system is overloaded whenever at least one resource is overloaded.
- * The class is used by the {@link AutoscaledPool} class.
+ * The class is used by the {@apilink AutoscaledPool} class.
  *
- * {@link SystemStatus.getCurrentStatus}
+ * {@apilink SystemStatus.getCurrentStatus}
  * returns a boolean that represents the current status of the system.
  * The length of the current timeframe in seconds is configurable
  * by the `currentHistorySecs` option and represents the max age
  * of snapshots to be considered for the calculation.
  *
- * {@link SystemStatus.getHistoricalStatus}
+ * {@apilink SystemStatus.getHistoricalStatus}
  * returns a boolean that represents the long-term status
  * of the system. It considers the full snapshot history available
- * in the {@link Snapshotter} instance.
+ * in the {@apilink Snapshotter} instance.
  * @category Scaling
  */
 export declare class SystemStatus {
@@ -112,7 +112,7 @@ export declare class SystemStatus {
     private readonly snapshotter;
     constructor(options?: SystemStatusOptions);
     /**
-     * Returns an {@link SystemInfo} object with the following structure:
+     * Returns an {@apilink SystemInfo} object with the following structure:
      *
      * ```javascript
      * {
@@ -129,7 +129,7 @@ export declare class SystemStatus {
      */
     getCurrentStatus(): SystemInfo;
     /**
-     * Returns an {@link SystemInfo} object with the following structure:
+     * Returns an {@apilink SystemInfo} object with the following structure:
      *
      * ```javascript
      * {
@@ -141,8 +141,8 @@ export declare class SystemStatus {
      * ```
      *
      * Where the `isSystemIdle` property is set to `false` if the system
-     * has been overloaded in the full history of the {@link Snapshotter}
-     * (which is configurable in the {@link Snapshotter}) and `true` otherwise.
+     * has been overloaded in the full history of the {@apilink Snapshotter}
+     * (which is configurable in the {@apilink Snapshotter}) and `true` otherwise.
      */
     getHistoricalStatus(): SystemInfo;
     /**

package/autoscaling/system_status.js

@@ -6,25 +6,25 @@ const ow_1 = tslib_1.__importDefault(require("ow"));
 const utils_1 = require("@crawlee/utils");
 const snapshotter_1 = require("./snapshotter");
 /**
- * Provides a simple interface to reading system status from a {@link Snapshotter} instance.
- * It only exposes two functions {@link SystemStatus.getCurrentStatus}
- * and {@link SystemStatus.getHistoricalStatus}.
+ * Provides a simple interface to reading system status from a {@apilink Snapshotter} instance.
+ * It only exposes two functions {@apilink SystemStatus.getCurrentStatus}
+ * and {@apilink SystemStatus.getHistoricalStatus}.
  * The system status is calculated using a weighted average of overloaded
  * messages in the snapshots, with the weights being the time intervals
  * between the snapshots. Each resource is calculated separately
  * and the system is overloaded whenever at least one resource is overloaded.
- * The class is used by the {@link AutoscaledPool} class.
+ * The class is used by the {@apilink AutoscaledPool} class.
  *
- * {@link SystemStatus.getCurrentStatus}
+ * {@apilink SystemStatus.getCurrentStatus}
  * returns a boolean that represents the current status of the system.
  * The length of the current timeframe in seconds is configurable
  * by the `currentHistorySecs` option and represents the max age
  * of snapshots to be considered for the calculation.
  *
- * {@link SystemStatus.getHistoricalStatus}
+ * {@apilink SystemStatus.getHistoricalStatus}
  * returns a boolean that represents the long-term status
  * of the system. It considers the full snapshot history available
- * in the {@link Snapshotter} instance.
+ * in the {@apilink Snapshotter} instance.
  * @category Scaling
  */
 class SystemStatus {
@@ -83,7 +83,7 @@ class SystemStatus {
         this.snapshotter = snapshotter || new snapshotter_1.Snapshotter({ config });
     }
     /**
-     * Returns an {@link SystemInfo} object with the following structure:
+     * Returns an {@apilink SystemInfo} object with the following structure:
      *
      * ```javascript
      * {
@@ -102,7 +102,7 @@ class SystemStatus {
         return this._isSystemIdle(this.currentHistorySecs);
     }
     /**
-     * Returns an {@link SystemInfo} object with the following structure:
+     * Returns an {@apilink SystemInfo} object with the following structure:
      *
      * ```javascript
      * {
@@ -114,8 +114,8 @@ class SystemStatus {
      * ```
      *
      * Where the `isSystemIdle` property is set to `false` if the system
-     * has been overloaded in the full history of the {@link Snapshotter}
-     * (which is configurable in the {@link Snapshotter}) and `true` otherwise.
+     * has been overloaded in the full history of the {@apilink Snapshotter}
+     * (which is configurable in the {@apilink Snapshotter}) and `true` otherwise.
      */
     getHistoricalStatus() {
         return this._isSystemIdle();

package/configuration.d.ts

@@ -6,44 +6,168 @@ import type { Dictionary, StorageClient } from '@crawlee/types';
 import { LogLevel } from '@apify/log';
 import type { EventManager } from './events';
 export interface ConfigurationOptions {
+    /**
+     * Defines storage client to be used.
+     * @default {@apilink MemoryStorage}
+     */
     storageClient?: StorageClient;
+    /**
+     * Defines the Event Manager to be used.
+     * @default {@apilink EventManager}
+     */
     eventManager?: EventManager;
+    /**
+     * Could be used to adjust the storage client behavior
+     * e.g. {@apilink MemoryStorageOptions} could be used to adjust the {@apilink MemoryStorage} behavior.
+     */
     storageClientOptions?: Dictionary;
+    /**
+     * Default dataset id.
+     *
+     * Alternative to `CRAWLEE_DEFAULT_DATASET_ID` environment variable.
+     * @default 'default'
+     */
     defaultDatasetId?: string;
+    /**
+     * Defines whether to purge the default storage folders before starting the crawler run.
+     *
+     * Alternative to `CRAWLEE_PURGE_ON_START` environment variable.
+     * @default true
+     */
     purgeOnStart?: boolean;
+    /**
+     * Default key-value store id.
+     *
+     * Alternative to `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` environment variable.
+     * @default 'default'
+     */
     defaultKeyValueStoreId?: string;
+    /**
+     * Default request queue id.
+     *
+     * Alternative to `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` environment variable.
+     * @default 'default'
+     */
     defaultRequestQueueId?: string;
+    /**
+     * Sets the ratio, defining the maximum CPU usage.
+     * When the CPU usage is higher than the provided ratio, the CPU is considered overloaded.
+     * @default 0.95
+     */
     maxUsedCpuRatio?: number;
+    /**
+     * Sets the ratio, defining the amount of system memory that could be used by the {@apilink AutoscaledPool}.
+     * When the memory usage is more than the provided ratio, the memory is considered overloaded.
+     *
+     * Alternative to `CRAWLEE_AVAILABLE_MEMORY_RATIO` environment variable.
+     * @default 0.25
+     */
     availableMemoryRatio?: number;
+    /**
+     * Sets the amount of system memory in megabytes to be used by the {@apilink AutoscaledPool}.
+     * By default, the maximum memory is set to one quarter of total system memory.
+     *
+     * Alternative to `CRAWLEE_MEMORY_MBYTES` environment variable.
+     */
     memoryMbytes?: number;
+    /**
+     * Defines the interval of emitting the `persistState` event.
+     *
+     * Alternative to `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` environment variable.
+     * @default 60_000
+     */
     persistStateIntervalMillis?: number;
+    /**
+     Defines the interval of emitting the `systemInfo` event.
+     @default 60_000
+     */
     systemInfoIntervalMillis?: number;
+    /**
+     * Defines the default input key, i.e. the key that is used to get the crawler input value
+     * from the default {@apilink KeyValueStore} associated with the current crawler run.
+     *
+     * Alternative to `CRAWLEE_INPUT_KEY` environment variable.
+     * @default 'INPUT'
+     */
     inputKey?: string;
+    /**
+     * Defines whether web browsers launched by Crawlee will run in the headless mode.
+     *
+     * Alternative to `CRAWLEE_HEADLESS` environment variable.
+     * @default true
+     */
     headless?: boolean;
+    /**
+     * Defines whether to run X virtual framebuffer on the web browsers launched by Crawlee.
+     *
+     * Alternative to `CRAWLEE_XVFB` environment variable.
+     * @default false
+     */
     xvfb?: boolean;
+    /**
+     * Defines a path to Chrome executable.
+     *
+     * Alternative to `CRAWLEE_CHROME_EXECUTABLE_PATH` environment variable.
+     */
     chromeExecutablePath?: string;
+    /**
+     * Defines a path to default browser executable.
+     *
+     * Alternative to `CRAWLEE_DEFAULT_BROWSER_PATH` environment variable.
+     */
     defaultBrowserPath?: string;
+    /**
+     * Defines whether to disable browser sandbox by adding `--no-sandbox` flag to `launchOptions`.
+     *
+     * Alternative to `CRAWLEE_DISABLE_BROWSER_SANDBOX` environment variable.
+     */
     disableBrowserSandbox?: boolean;
+    /**
+     * Sets the log level to the given value.
+     *
+     * Alternative to `CRAWLEE_LOG_LEVEL`.
+     * @default 'INFO'
+     */
     logLevel?: LogLevel | LogLevel[keyof LogLevel];
 }
 /**
  * `Configuration` is a value object holding Crawlee configuration. By default, there is a
  * global singleton instance of this class available via `Configuration.getGlobalConfig()`.
- * Places that depend on a configurable behaviour depend on this class as have the global
+ * Places that depend on a configurable behaviour depend on this class, as they have the global
  * instance as the default value.
  *
+ * *Using global configuration:*
  * ```js
  * import { BasicCrawler, Configuration } from 'crawlee';
  *
+ * // Get the global configuration
+ * const config = Configuration.getGlobalConfig();
+ * // Set the 'persistStateIntervalMillis' option
+ * // of global configuration to 10 seconds
+ * config.set('persistStateIntervalMillis', 10_000);
+ *
+ * // No need to pass the configuration to the crawler,
+ * // as it's using the global configuration by default
+ * const crawler = new BasicCrawler();
+ * ```
+ *
+ * *Using custom configuration:*
+ * ```js
+ * import { BasicCrawler, Configuration } from 'crawlee';
+ *
+ * // Create a new configuration
  * const config = new Configuration({ persistStateIntervalMillis: 30_000 });
+ * // Pass the configuration to the crawler
  * const crawler = new BasicCrawler({ ... }, config);
  * ```
  *
  * The configuration provided via environment variables always takes precedence. We can also
  * define the `crawlee.json` file in the project root directory which will serve as a baseline,
- * so the options provided in constructor will override those.
+ * so the options provided in constructor will override those. In other words, the precedence is:
  *
- * > In other words, the precedence is: crawlee.json < constructor options < environment variables.
+ * ```text
+ * crawlee.json < constructor options < environment variables
+ * ```
  *
  * ## Supported Configuration Options
  *
@@ -100,8 +224,8 @@ export declare class Configuration {
      */
     set(key: keyof ConfigurationOptions, value?: any): void;
     /**
-     * Returns cached instance of {@link StorageClient} using options as defined in the environment variables or in
-     * this {@link Configuration} instance. Only first call of this method will create the client, following calls will
+     * Returns cached instance of {@apilink StorageClient} using options as defined in the environment variables or in
+     * this {@apilink Configuration} instance. Only first call of this method will create the client, following calls will
      * return the same client instance.
      *
      * Caching works based on the `storageClientOptions`, so calling this method with different options will return
@@ -122,11 +246,11 @@ export declare class Configuration {
      */
     static getGlobalConfig(): Configuration;
     /**
-     * Gets default {@link StorageClient} instance.
+     * Gets default {@apilink StorageClient} instance.
      */
     static getStorageClient(): StorageClient;
     /**
-     * Gets default {@link EventManager} instance.
+     * Gets default {@apilink EventManager} instance.
      */
     static getEventManager(): EventManager;
     /**

package/configuration.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"configuration.d.ts","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAGxD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAErD,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAChE,OAAY,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAG7C,MAAM,WAAW,oBAAoB;IACjC,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,oBAAoB,CAAC,EAAE,UAAU,CAAC;IAClC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC,MAAM,QAAQ,CAAC,CAAC;CAClD;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,qBAAa,aAAa;IACtB;;OAEG;IACH,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,UAAU,CAelC;IAEF,SAAS,CAAC,MAAM,CAAC,YAAY,WAAiE;IAE9F,SAAS,CAAC,MAAM,CAAC,YAAY,WAA8E;IAE3G,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,UAAU,CAYnC;IAEF;;;OAGG;IACH,MAAM,CAAC,OAAO,mCAA0C;IAExD,SAAS,CAAC,OAAO,EAAG,GAAG,CAAC,MAAM,oBAAoB,EAAE,oBAAoB,CAAC,MAAM,oBAAoB,CAAC,CAAC,CAAC;IACtG,SAAS,CAAC,QAAQ,uBAA8B;IAEhD,gBAAgB;IAChB,MAAM,CAAC,YAAY,CAAC,EAAE,aAAa,CAAC;IAEpC;;OAEG;gBACS,OAAO,GAAE,oBAAyB;IAe9C;;;;OAIG;IACH,GAAG,CAAC,CAAC,SAAS,MAAM,oBAAoB,EAAE,CAAC,SAAS,oBAAoB,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC;IA2BzG,SAAS,CAAC,aAAa,CAAC,GAAG,EAAE,MAAM,oBAAoB,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO;IAazF;;;OAGG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,oBAAoB,EAAE,KAAK,CAAC,EAAE,GAAG,GAAG,IAAI;IAIvD;;;;;;;;OAQG;IACH,gBAAgB,IAAI,aAAa;IASjC,eAAe,IAAI,YAAY;IAe/B;;;OAGG;IACH,mBAAmB,CAAC,OAAO,GAAE,oBAAyB,GAAG,aAAa;IAatE,gBAAgB,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI;IAI7C,eAAe,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;IAI3C;;OAEG;IACH,MAAM,CAAC,eAAe,IAAI,aAAa;IASvC;;OAEG;IACH,MAAM,CAAC,gBAAgB,IAAI,aAAa;IAIxC;;OAEG;IACH,MAAM,CAAC,eAAe,IAAI,YAAY;IAItC;;;OAGG;IACH,MAAM,CAAC,gBAAgB,IAAI,IAAI;IAI/B,SAAS,CAAC,YAAY,CAAC,OAAO,EAAE,oBAAoB;CAgBvD"}
+{"version":3,"file":"configuration.d.ts","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAGxD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAErD,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAChE,OAAY,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAG7C,MAAM,WAAW,oBAAoB;IACjC;;;OAGG;IACH,aAAa,CAAC,EAAE,aAAa,CAAC;IAE9B;;;OAGG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B;;;OAGG;IACH,oBAAoB,CAAC,EAAE,UAAU,CAAC;IAElC;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;;OAKG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IAEvB;;;;;OAKG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAEhC;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;;;OAMG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAE9B;;;;;OAKG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;;;;OAKG;IACH,0BAA0B,CAAC,EAAE,MAAM,CAAC;IAEpC;;;OAGG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;;;;OAKG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAE9B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAEhC;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC,MAAM,QAAQ,CAAC,CAAC;CAClD;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2DG;AACH,qBAAa,aAAa;IACtB;;OAEG;IACH,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,UAAU,CAelC;IAEF,SAAS,CAAC,MAAM,CAAC,YAAY,WAAiE;IAE9F,SAAS,CAAC,MAAM,CAAC,YAAY,WAA8E;IAE3G,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,UAAU,CAYnC;IAEF;;;OAGG;IACH,MAAM,CAAC,OAAO,mCAA0C;IAExD,SAAS,CAAC,OAAO,EAAG,GAAG,CAAC,MAAM,oBAAoB,EAAE,oBAAoB,CAAC,MAAM,oBAAoB,CAAC,CAAC,CAAC;IACtG,SAAS,CAAC,QAAQ,uBAA8B;IAEhD,gBAAgB;IAChB,MAAM,CAAC,YAAY,CAAC,EAAE,aAAa,CAAC;IAEpC;;OAEG;gBACS,OAAO,GAAE,oBAAyB;IAe9C;;;;OAIG;IACH,GAAG,CAAC,CAAC,SAAS,MAAM,oBAAoB,EAAE,CAAC,SAAS,oBAAoB,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC;IA2BzG,SAAS,CAAC,aAAa,CAAC,GAAG,EAAE,MAAM,oBAAoB,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO;IAazF;;;OAGG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,oBAAoB,EAAE,KAAK,CAAC,EAAE,GAAG,GAAG,IAAI;IAIvD;;;;;;;;OAQG;IACH,gBAAgB,IAAI,aAAa;IASjC,eAAe,IAAI,YAAY;IAe/B;;;OAGG;IACH,mBAAmB,CAAC,OAAO,GAAE,oBAAyB,GAAG,aAAa;IAatE,gBAAgB,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI;IAI7C,eAAe,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;IAI3C;;OAEG;IACH,MAAM,CAAC,eAAe,IAAI,aAAa;IASvC;;OAEG;IACH,MAAM,CAAC,gBAAgB,IAAI,aAAa;IAIxC;;OAEG;IACH,MAAM,CAAC,eAAe,IAAI,YAAY;IAItC;;;OAGG;IACH,MAAM,CAAC,gBAAgB,IAAI,IAAI;IAI/B,SAAS,CAAC,YAAY,CAAC,OAAO,EAAE,oBAAoB;CAgBvD"}

package/configuration.js

@@ -13,21 +13,41 @@ const events_1 = require("./events");
 /**
  * `Configuration` is a value object holding Crawlee configuration. By default, there is a
  * global singleton instance of this class available via `Configuration.getGlobalConfig()`.
- * Places that depend on a configurable behaviour depend on this class as have the global
+ * Places that depend on a configurable behaviour depend on this class, as they have the global
  * instance as the default value.
  *
+ * *Using global configuration:*
  * ```js
  * import { BasicCrawler, Configuration } from 'crawlee';
  *
+ * // Get the global configuration
+ * const config = Configuration.getGlobalConfig();
+ * // Set the 'persistStateIntervalMillis' option
+ * // of global configuration to 10 seconds
+ * config.set('persistStateIntervalMillis', 10_000);
+ *
+ * // No need to pass the configuration to the crawler,
+ * // as it's using the global configuration by default
+ * const crawler = new BasicCrawler();
+ * ```
+ *
+ * *Using custom configuration:*
+ * ```js
+ * import { BasicCrawler, Configuration } from 'crawlee';
+ *
+ * // Create a new configuration
  * const config = new Configuration({ persistStateIntervalMillis: 30_000 });
+ * // Pass the configuration to the crawler
  * const crawler = new BasicCrawler({ ... }, config);
  * ```
  *
  * The configuration provided via environment variables always takes precedence. We can also
  * define the `crawlee.json` file in the project root directory which will serve as a baseline,
- * so the options provided in constructor will override those.
+ * so the options provided in constructor will override those. In other words, the precedence is:
  *
- * > In other words, the precedence is: crawlee.json < constructor options < environment variables.
+ * ```text
+ * crawlee.json < constructor options < environment variables
+ * ```
  *
  * ## Supported Configuration Options
  *
@@ -121,8 +141,8 @@ class Configuration {
         this.options.set(key, value);
     }
     /**
-     * Returns cached instance of {@link StorageClient} using options as defined in the environment variables or in
-     * this {@link Configuration} instance. Only first call of this method will create the client, following calls will
+     * Returns cached instance of {@apilink StorageClient} using options as defined in the environment variables or in
+     * this {@apilink Configuration} instance. Only first call of this method will create the client, following calls will
      * return the same client instance.
      *
      * Caching works based on the `storageClientOptions`, so calling this method with different options will return
@@ -177,13 +197,13 @@ class Configuration {
         return Configuration.globalConfig;
     }
     /**
-     * Gets default {@link StorageClient} instance.
+     * Gets default {@apilink StorageClient} instance.
      */
     static getStorageClient() {
         return this.getGlobalConfig().getStorageClient();
     }
     /**
-     * Gets default {@link EventManager} instance.
+     * Gets default {@apilink EventManager} instance.
      */
     static getEventManager() {
         return this.getGlobalConfig().getEventManager();

package/configuration.js.map

@@ -1 +1 @@
-{"version":3,"file":"configuration.js","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":";;;;AACA,4DAAwD;AACxD,uCAAwD;AACxD,yCAAiC;AACjC,uDAAqD;AACrD,6CAA2C;AAE3C,0DAA2C;AAC3C,yCAAqC;AAErC,qCAA6C;AAwB7C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,MAAa,aAAa;IAmDtB;;OAEG;IACH,YAAY,UAAgC,EAAE;QAT9C;;;;;WAAsG;QACtG;;;;mBAAqB,IAAI,GAAG,EAAmB;WAAC;QAS5C,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QAE3B,oEAAoE;QACpE,0BAAY,CAAC,mBAAmB,GAAG,EAAE,CAAC;QAEtC,6DAA6D;QAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAEtC,IAAI,QAAQ,EAAE;YACV,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,cAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAyB,CAAC,CAAC;YACvH,aAAG,CAAC,QAAQ,CAAC,KAAiB,CAAC,CAAC;SACnC;IACL,CAAC;IAED;;;;OAIG;IACH,GAAG,CAA0E,GAAM,EAAE,YAAgB;QACjG,iHAAiH;QACjH,IAAI,QAA4B,CAAC;QAEjC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,IAAA,kBAAO,EAAC,aAAa,CAAC,OAAO,CAAC,EAAE;YACjD,IAAI,GAAG,KAAK,CAAC,EAAE;gBACX,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAW,CAAC,CAAC;gBAEpC,IAAI,QAAQ,EAAE;oBACV,MAAM;iBACT;aACJ;SACJ;QAED,IAAI,QAAQ,IAAI,IAAI,EAAE;YAClB,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,QAAQ,CAAM,CAAC;SACjD;QAED,+BAA+B;QAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YACvB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;SACrC;QAED,uBAAuB;QACvB,OAAO,CAAC,YAAY,IAAI,aAAa,CAAC,QAAQ,CAAC,GAA0C,CAAC,IAAI,QAAQ,CAAM,CAAC;IACjH,CAAC;IAES,aAAa,CAAC,GAA+B,EAAE,KAAgC;QACrF,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;YAC1C,OAAO,CAAC,KAAK,CAAC;SACjB;QAED,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;YAC1C,wDAAwD;YACxD,OAAO,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;SACpE;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;OAGG;IACH,GAAG,CAAC,GAA+B,EAAE,KAAW;QAC5C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACjC,CAAC;IAED;;;;;;;;OAQG;IACH,gBAAgB;QACZ,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE;YACnC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAkB,CAAC;SAC7D;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAe,CAAC;QACvE,OAAO,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED,eAAe;QACX,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE;YAClC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAiB,CAAC;SAC3D;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE;YACnC,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAiB,CAAC;SAC5D;QAED,MAAM,YAAY,GAAG,IAAI,0BAAiB,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;QAEhD,OAAO,YAAY,CAAC;IACxB,CAAC;IAED;;;OAGG;IACH,mBAAmB,CAAC,UAAgC,EAAE;QAClD,MAAM,QAAQ,GAAG,iBAAiB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;QAE5D,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;YAC7B,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAkB,CAAC;SACvD;QAED,MAAM,OAAO,GAAG,IAAI,8BAAa,CAAC,OAAO,CAAC,CAAC;QAC3C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAErC,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,gBAAgB,CAAC,MAAqB;QAClC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAC9C,CAAC;IAED,eAAe,CAAC,MAAoB;QAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe;QAClB,IAAI,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE;YAClC,OAAO,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAG,CAAC;SAC5C;QAED,aAAa,CAAC,YAAY,KAA1B,aAAa,CAAC,YAAY,GAAK,IAAI,aAAa,EAAE,EAAC;QACnD,OAAO,aAAa,CAAC,YAAY,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,gBAAgB;QACnB,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC,gBAAgB,EAAE,CAAC;IACrD,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe;QAClB,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC,eAAe,EAAE,CAAC;IACpD,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,gBAAgB;QACnB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAES,YAAY,CAAC,OAA6B;QAChD,8DAA8D;QAC9D,MAAM,IAAI,GAAG,IAAA,gBAAI,EAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,CAAC,CAAC;QAEjD,IAAI,IAAA,yBAAc,EAAC,IAAI,CAAC,EAAE;YACtB,IAAI;gBACA,MAAM,IAAI,GAAG,IAAA,uBAAY,EAAC,IAAI,CAAC,CAAC;gBAChC,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC1D,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;aACjD;YAAC,MAAM;gBACJ,SAAS;aACZ;SACJ;QAED,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,CAAC,IAAA,kBAAO,EAAC,OAAO,CAAC,CAAC,CAAC;IAC7C,CAAC;;AArOL,sCAsOC;AArOG;;GAEG;AACH;;;;WAAuC;QACnC,8BAA8B,EAAE,sBAAsB;QACtD,sBAAsB,EAAE,cAAc;QACtC,qBAAqB,EAAE,cAAc;QACrC,0BAA0B,EAAE,kBAAkB;QAC9C,kCAAkC,EAAE,wBAAwB;QAC5D,gCAAgC,EAAE,uBAAuB;QACzD,iBAAiB,EAAE,UAAU;QAC7B,qCAAqC,EAAE,4BAA4B;QACnE,gBAAgB,EAAE,UAAU;QAC5B,YAAY,EAAE,MAAM;QACpB,8BAA8B,EAAE,sBAAsB;QACtD,4BAA4B,EAAE,oBAAoB;QAClD,+BAA+B,EAAE,uBAAuB;QACxD,iBAAiB,EAAE,UAAU;KAChC;GAAC;AAEF;;;;WAAgC,CAAC,cAAc,EAAE,UAAU,EAAE,MAAM,EAAE,uBAAuB,CAAC;GAAC;AAE9F;;;;WAAgC,CAAC,cAAc,EAAE,4BAA4B,EAAE,0BAA0B,CAAC;GAAC;AAE3G;;;;WAAwC;QACpC,sBAAsB,EAAE,SAAS;QACjC,gBAAgB,EAAE,SAAS;QAC3B,qBAAqB,EAAE,SAAS;QAChC,QAAQ,EAAE,OAAO;QACjB,eAAe,EAAE,IAAI;QACrB,oBAAoB,EAAE,IAAI;QAC1B,oBAAoB,EAAE,EAAE;QACxB,YAAY,EAAE,IAAI;QAClB,QAAQ,EAAE,IAAI;QACd,0BAA0B,EAAE,KAAM;QAClC,wBAAwB,EAAE,KAAM;KACnC;GAAC;AAEF;;;GAGG;AACH;;;;WAAiB,IAAI,oCAAiB,EAAiB;GAAC"}
+{"version":3,"file":"configuration.js","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":";;;;AACA,4DAAwD;AACxD,uCAAwD;AACxD,yCAAiC;AACjC,uDAAqD;AACrD,6CAA2C;AAE3C,0DAA2C;AAC3C,yCAAqC;AAErC,qCAA6C;AAkJ7C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2DG;AACH,MAAa,aAAa;IAmDtB;;OAEG;IACH,YAAY,UAAgC,EAAE;QAT9C;;;;;WAAsG;QACtG;;;;mBAAqB,IAAI,GAAG,EAAmB;WAAC;QAS5C,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QAE3B,oEAAoE;QACpE,0BAAY,CAAC,mBAAmB,GAAG,EAAE,CAAC;QAEtC,6DAA6D;QAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAEtC,IAAI,QAAQ,EAAE;YACV,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,cAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAyB,CAAC,CAAC;YACvH,aAAG,CAAC,QAAQ,CAAC,KAAiB,CAAC,CAAC;SACnC;IACL,CAAC;IAED;;;;OAIG;IACH,GAAG,CAA0E,GAAM,EAAE,YAAgB;QACjG,iHAAiH;QACjH,IAAI,QAA4B,CAAC;QAEjC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,IAAA,kBAAO,EAAC,aAAa,CAAC,OAAO,CAAC,EAAE;YACjD,IAAI,GAAG,KAAK,CAAC,EAAE;gBACX,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAW,CAAC,CAAC;gBAEpC,IAAI,QAAQ,EAAE;oBACV,MAAM;iBACT;aACJ;SACJ;QAED,IAAI,QAAQ,IAAI,IAAI,EAAE;YAClB,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,QAAQ,CAAM,CAAC;SACjD;QAED,+BAA+B;QAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YACvB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;SACrC;QAED,uBAAuB;QACvB,OAAO,CAAC,YAAY,IAAI,aAAa,CAAC,QAAQ,CAAC,GAA0C,CAAC,IAAI,QAAQ,CAAM,CAAC;IACjH,CAAC;IAES,aAAa,CAAC,GAA+B,EAAE,KAAgC;QACrF,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;YAC1C,OAAO,CAAC,KAAK,CAAC;SACjB;QAED,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;YAC1C,wDAAwD;YACxD,OAAO,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;SACpE;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;OAGG;IACH,GAAG,CAAC,GAA+B,EAAE,KAAW;QAC5C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACjC,CAAC;IAED;;;;;;;;OAQG;IACH,gBAAgB;QACZ,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE;YACnC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAkB,CAAC;SAC7D;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAe,CAAC;QACvE,OAAO,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED,eAAe;QACX,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE;YAClC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAiB,CAAC;SAC3D;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE;YACnC,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAiB,CAAC;SAC5D;QAED,MAAM,YAAY,GAAG,IAAI,0BAAiB,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;QAEhD,OAAO,YAAY,CAAC;IACxB,CAAC;IAED;;;OAGG;IACH,mBAAmB,CAAC,UAAgC,EAAE;QAClD,MAAM,QAAQ,GAAG,iBAAiB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;QAE5D,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;YAC7B,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAkB,CAAC;SACvD;QAED,MAAM,OAAO,GAAG,IAAI,8BAAa,CAAC,OAAO,CAAC,CAAC;QAC3C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAErC,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,gBAAgB,CAAC,MAAqB;QAClC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAC9C,CAAC;IAED,eAAe,CAAC,MAAoB;QAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe;QAClB,IAAI,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE;YAClC,OAAO,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAG,CAAC;SAC5C;QAED,aAAa,CAAC,YAAY,KAA1B,aAAa,CAAC,YAAY,GAAK,IAAI,aAAa,EAAE,EAAC;QACnD,OAAO,aAAa,CAAC,YAAY,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,gBAAgB;QACnB,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC,gBAAgB,EAAE,CAAC;IACrD,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe;QAClB,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC,eAAe,EAAE,CAAC;IACpD,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,gBAAgB;QACnB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAES,YAAY,CAAC,OAA6B;QAChD,8DAA8D;QAC9D,MAAM,IAAI,GAAG,IAAA,gBAAI,EAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,CAAC,CAAC;QAEjD,IAAI,IAAA,yBAAc,EAAC,IAAI,CAAC,EAAE;YACtB,IAAI;gBACA,MAAM,IAAI,GAAG,IAAA,uBAAY,EAAC,IAAI,CAAC,CAAC;gBAChC,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC1D,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;aACjD;YAAC,MAAM;gBACJ,SAAS;aACZ;SACJ;QAED,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,CAAC,IAAA,kBAAO,EAAC,OAAO,CAAC,CAAC,CAAC;IAC7C,CAAC;;AArOL,sCAsOC;AArOG;;GAEG;AACH;;;;WAAuC;QACnC,8BAA8B,EAAE,sBAAsB;QACtD,sBAAsB,EAAE,cAAc;QACtC,qBAAqB,EAAE,cAAc;QACrC,0BAA0B,EAAE,kBAAkB;QAC9C,kCAAkC,EAAE,wBAAwB;QAC5D,gCAAgC,EAAE,uBAAuB;QACzD,iBAAiB,EAAE,UAAU;QAC7B,qCAAqC,EAAE,4BAA4B;QACnE,gBAAgB,EAAE,UAAU;QAC5B,YAAY,EAAE,MAAM;QACpB,8BAA8B,EAAE,sBAAsB;QACtD,4BAA4B,EAAE,oBAAoB;QAClD,+BAA+B,EAAE,uBAAuB;QACxD,iBAAiB,EAAE,UAAU;KAChC;GAAC;AAEF;;;;WAAgC,CAAC,cAAc,EAAE,UAAU,EAAE,MAAM,EAAE,uBAAuB,CAAC;GAAC;AAE9F;;;;WAAgC,CAAC,cAAc,EAAE,4BAA4B,EAAE,0BAA0B,CAAC;GAAC;AAE3G;;;;WAAwC;QACpC,sBAAsB,EAAE,SAAS;QACjC,gBAAgB,EAAE,SAAS;QAC3B,qBAAqB,EAAE,SAAS;QAChC,QAAQ,EAAE,OAAO;QACjB,eAAe,EAAE,IAAI;QACrB,oBAAoB,EAAE,IAAI;QAC1B,oBAAoB,EAAE,EAAE;QACxB,YAAY,EAAE,IAAI;QAClB,QAAQ,EAAE,IAAI;QACd,0BAA0B,EAAE,KAAM;QAClC,wBAAwB,EAAE,KAAM;KACnC;GAAC;AAEF;;;GAGG;AACH;;;;WAAiB,IAAI,oCAAiB,EAAiB;GAAC"}

package/crawlers/crawler_commons.d.ts

@@ -6,13 +6,13 @@ import type { Session } from '../session_pool/session';
 export interface CrawlingContext<UserData extends Dictionary = Dictionary> extends Record<PropertyKey, unknown> {
     id: string;
     /**
-     * The original {@link Request} object.
+     * The original {@apilink Request} object.
      */
     request: Request<UserData>;
     session?: Session;
     /**
      * An object with information about currently used proxy by the crawler
-     * and configured by the {@link ProxyConfiguration} class.
+     * and configured by the {@apilink ProxyConfiguration} class.
      */
     proxyInfo?: ProxyInfo;
     log: Log;

package/crawlers/statistics.d.ts

@@ -28,7 +28,7 @@ export declare class Statistics {
      */
     readonly id: number;
     /**
-     * Current statistic state used for doing calculations on {@link Statistics.calculate} calls
+     * Current statistic state used for doing calculations on {@apilink Statistics.calculate} calls
      */
     state: StatisticState;
     /**

package/crawlers/statistics.js

@@ -67,7 +67,7 @@ class Statistics {
             value: Statistics.id++
         }); // assign an id while incrementing so it can be saved/restored from KV
         /**
-         * Current statistic state used for doing calculations on {@link Statistics.calculate} calls
+         * Current statistic state used for doing calculations on {@apilink Statistics.calculate} calls
          */
         Object.defineProperty(this, "state", {
             enumerable: true,