@crawlee/core 3.13.3-beta.9 → 3.13.4-beta.0

package/autoscaling/autoscaled_pool.d.ts

@@ -20,7 +20,7 @@ export interface AutoscaledPoolOptions {
      * If it resolves to `true` then the pool's run finishes. Being called only
      * when there are no tasks being processed means that as long as `isTaskReadyFunction()`
      * keeps resolving to `true`, `isFinishedFunction()` will never be called.
-     * To abort a run, use the {@apilink AutoscaledPool.abort} method.
+     * To abort a run, use the {@link AutoscaledPool.abort} method.
      */
     isFinishedFunction?: () => Promise<boolean>;
     /**
@@ -84,12 +84,12 @@ export interface AutoscaledPoolOptions {
      */
     taskTimeoutSecs?: number;
     /**
-     * Options to be passed down to the {@apilink Snapshotter} constructor. This is useful for fine-tuning
+     * Options to be passed down to the {@link Snapshotter} constructor. This is useful for fine-tuning
      * the snapshot intervals and history.
      */
     snapshotterOptions?: SnapshotterOptions;
     /**
-     * Options to be passed down to the {@apilink SystemStatus} constructor. This is useful for fine-tuning
+     * Options to be passed down to the {@link SystemStatus} constructor. This is useful for fine-tuning
      * the system status reports. If a custom snapshotter is set in the options, it will be used
      * by the pool.
      */
@@ -106,22 +106,22 @@ export interface AutoscaledPoolOptions {
  * The pool only starts new tasks if there is enough free CPU and memory available
  * and the Javascript event loop is not blocked.
  *
- * The information about the CPU and memory usage is obtained by the {@apilink Snapshotter} class,
+ * The information about the CPU and memory usage is obtained by the {@link Snapshotter} class,
  * which makes regular snapshots of system resources that may be either local
  * or from the Apify cloud infrastructure in case the process is running on the Apify platform.
- * Meaningful data gathered from these snapshots is provided to `AutoscaledPool` by the {@apilink SystemStatus} class.
+ * Meaningful data gathered from these snapshots is provided to `AutoscaledPool` by the {@link SystemStatus} class.
  *
  * Before running the pool, you need to implement the following three functions:
- * {@apilink AutoscaledPoolOptions.runTaskFunction},
- * {@apilink AutoscaledPoolOptions.isTaskReadyFunction} and
- * {@apilink AutoscaledPoolOptions.isFinishedFunction}.
+ * {@link AutoscaledPoolOptions.runTaskFunction},
+ * {@link AutoscaledPoolOptions.isTaskReadyFunction} and
+ * {@link AutoscaledPoolOptions.isFinishedFunction}.
  *
- * The auto-scaled pool is started by calling the {@apilink AutoscaledPool.run} function.
- * The pool periodically queries the {@apilink AutoscaledPoolOptions.isTaskReadyFunction} function
+ * The auto-scaled pool is started by calling the {@link AutoscaledPool.run} function.
+ * The pool periodically queries the {@link AutoscaledPoolOptions.isTaskReadyFunction} function
  * for more tasks, managing optimal concurrency, until the function resolves to `false`. The pool then queries
- * the {@apilink AutoscaledPoolOptions.isFinishedFunction}. If it resolves to `true`, the run finishes after all running tasks complete.
+ * the {@link AutoscaledPoolOptions.isFinishedFunction}. If it resolves to `true`, the run finishes after all running tasks complete.
  * If it resolves to `false`, it assumes there will be more tasks available later and keeps periodically querying for tasks.
- * If any of the tasks throws then the {@apilink AutoscaledPool.run} function rejects the promise with an error.
+ * If any of the tasks throws then the {@link AutoscaledPool.run} function rejects the promise with an error.
  *
  * The pool evaluates whether it should start a new task every time one of the tasks finishes
  * and also in the interval set by the `options.maybeRunIntervalSecs` parameter.
@@ -220,7 +220,7 @@ export declare class AutoscaledPool {
     run(): Promise<void>;
     /**
      * Aborts the run of the auto-scaled pool and destroys it. The promise returned from
-     * the {@apilink AutoscaledPool.run} function will immediately resolve, no more new tasks
+     * the {@link AutoscaledPool.run} function will immediately resolve, no more new tasks
      * will be spawned and all running tasks will be left in their current state.
      *
      * Due to the nature of the tasks, auto-scaled pool cannot reliably guarantee abortion
@@ -232,19 +232,19 @@ export declare class AutoscaledPool {
     abort(): Promise<void>;
     /**
      * Prevents the auto-scaled pool from starting new tasks, but allows the running ones to finish
-     * (unlike abort, which terminates them). Used together with {@apilink AutoscaledPool.resume}
+     * (unlike abort, which terminates them). Used together with {@link AutoscaledPool.resume}
      *
      * The function's promise will resolve once all running tasks have completed and the pool
      * is effectively idle. If the `timeoutSecs` argument is provided, the promise will reject
      * with a timeout error after the `timeoutSecs` seconds.
      *
-     * The promise returned from the {@apilink AutoscaledPool.run} function will not resolve
+     * The promise returned from the {@link AutoscaledPool.run} function will not resolve
      * when `.pause()` is invoked (unlike abort, which resolves it).
      */
     pause(timeoutSecs?: number): Promise<void>;
     /**
      * Resumes the operation of the autoscaled-pool by allowing more tasks to be run.
-     * Used together with {@apilink AutoscaledPool.pause}
+     * Used together with {@link AutoscaledPool.pause}
      *
      * Tasks will automatically start running again in `options.maybeRunIntervalSecs`.
      */

package/autoscaling/autoscaled_pool.js

@@ -15,22 +15,22 @@ const system_status_1 = require("./system_status");
  * The pool only starts new tasks if there is enough free CPU and memory available
  * and the Javascript event loop is not blocked.
  *
- * The information about the CPU and memory usage is obtained by the {@apilink Snapshotter} class,
+ * The information about the CPU and memory usage is obtained by the {@link Snapshotter} class,
  * which makes regular snapshots of system resources that may be either local
  * or from the Apify cloud infrastructure in case the process is running on the Apify platform.
- * Meaningful data gathered from these snapshots is provided to `AutoscaledPool` by the {@apilink SystemStatus} class.
+ * Meaningful data gathered from these snapshots is provided to `AutoscaledPool` by the {@link SystemStatus} class.
  *
  * Before running the pool, you need to implement the following three functions:
- * {@apilink AutoscaledPoolOptions.runTaskFunction},
- * {@apilink AutoscaledPoolOptions.isTaskReadyFunction} and
- * {@apilink AutoscaledPoolOptions.isFinishedFunction}.
+ * {@link AutoscaledPoolOptions.runTaskFunction},
+ * {@link AutoscaledPoolOptions.isTaskReadyFunction} and
+ * {@link AutoscaledPoolOptions.isFinishedFunction}.
  *
- * The auto-scaled pool is started by calling the {@apilink AutoscaledPool.run} function.
- * The pool periodically queries the {@apilink AutoscaledPoolOptions.isTaskReadyFunction} function
+ * The auto-scaled pool is started by calling the {@link AutoscaledPool.run} function.
+ * The pool periodically queries the {@link AutoscaledPoolOptions.isTaskReadyFunction} function
  * for more tasks, managing optimal concurrency, until the function resolves to `false`. The pool then queries
- * the {@apilink AutoscaledPoolOptions.isFinishedFunction}. If it resolves to `true`, the run finishes after all running tasks complete.
+ * the {@link AutoscaledPoolOptions.isFinishedFunction}. If it resolves to `true`, the run finishes after all running tasks complete.
  * If it resolves to `false`, it assumes there will be more tasks available later and keeps periodically querying for tasks.
- * If any of the tasks throws then the {@apilink AutoscaledPool.run} function rejects the promise with an error.
+ * If any of the tasks throws then the {@link AutoscaledPool.run} function rejects the promise with an error.
  *
  * The pool evaluates whether it should start a new task every time one of the tasks finishes
  * and also in the interval set by the `options.maybeRunIntervalSecs` parameter.
@@ -373,7 +373,7 @@ class AutoscaledPool {
     }
     /**
      * Aborts the run of the auto-scaled pool and destroys it. The promise returned from
-     * the {@apilink AutoscaledPool.run} function will immediately resolve, no more new tasks
+     * the {@link AutoscaledPool.run} function will immediately resolve, no more new tasks
      * will be spawned and all running tasks will be left in their current state.
      *
      * Due to the nature of the tasks, auto-scaled pool cannot reliably guarantee abortion
@@ -391,13 +391,13 @@ class AutoscaledPool {
     }
     /**
      * Prevents the auto-scaled pool from starting new tasks, but allows the running ones to finish
-     * (unlike abort, which terminates them). Used together with {@apilink AutoscaledPool.resume}
+     * (unlike abort, which terminates them). Used together with {@link AutoscaledPool.resume}
      *
      * The function's promise will resolve once all running tasks have completed and the pool
      * is effectively idle. If the `timeoutSecs` argument is provided, the promise will reject
      * with a timeout error after the `timeoutSecs` seconds.
      *
-     * The promise returned from the {@apilink AutoscaledPool.run} function will not resolve
+     * The promise returned from the {@link AutoscaledPool.run} function will not resolve
      * when `.pause()` is invoked (unlike abort, which resolves it).
      */
     async pause(timeoutSecs) {
@@ -426,7 +426,7 @@ class AutoscaledPool {
     }
     /**
      * Resumes the operation of the autoscaled-pool by allowing more tasks to be run.
-     * Used together with {@apilink AutoscaledPool.pause}
+     * Used together with {@link AutoscaledPool.pause}
      *
      * Tasks will automatically start running again in `options.maybeRunIntervalSecs`.
      */

package/autoscaling/snapshotter.d.ts

@@ -75,7 +75,7 @@ interface ClientSnapshot {
  * Creates snapshots of system resources at given intervals and marks the resource
  * as either overloaded or not during the last interval. Keeps a history of the snapshots.
  * It tracks the following resources: Memory, EventLoop, API and CPU.
- * The class is used by the {@apilink AutoscaledPool} class.
+ * The class is used by the {@link AutoscaledPool} class.
  *
  * When running on the Apify platform, the CPU and memory statistics are provided by the platform,
  * as collected from the running Docker container. When running locally, `Snapshotter`

package/autoscaling/snapshotter.js

@@ -14,7 +14,7 @@ const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10000;
  * Creates snapshots of system resources at given intervals and marks the resource
  * as either overloaded or not during the last interval. Keeps a history of the snapshots.
  * It tracks the following resources: Memory, EventLoop, API and CPU.
- * The class is used by the {@apilink AutoscaledPool} class.
+ * The class is used by the {@link AutoscaledPool} class.
  *
  * When running on the Apify platform, the CPU and memory statistics are provided by the platform,
  * as collected from the running Docker container. When running locally, `Snapshotter`

package/autoscaling/system_status.d.ts

@@ -29,7 +29,7 @@ export interface SystemInfo {
 }
 export interface SystemStatusOptions {
     /**
-     * Defines max age of snapshots used in the {@apilink SystemStatus.getCurrentStatus} measurement.
+     * Defines max age of snapshots used in the {@link SystemStatus.getCurrentStatus} measurement.
      * @default 5
      */
     currentHistorySecs?: number;
@@ -82,25 +82,25 @@ export interface FinalStatistics {
     crawlerRuntimeMillis: number;
 }
 /**
- * Provides a simple interface to reading system status from a {@apilink Snapshotter} instance.
- * It only exposes two functions {@apilink SystemStatus.getCurrentStatus}
- * and {@apilink SystemStatus.getHistoricalStatus}.
+ * Provides a simple interface to reading system status from a {@link Snapshotter} instance.
+ * It only exposes two functions {@link SystemStatus.getCurrentStatus}
+ * and {@link SystemStatus.getHistoricalStatus}.
  * The system status is calculated using a weighted average of overloaded
  * messages in the snapshots, with the weights being the time intervals
  * between the snapshots. Each resource is calculated separately
  * and the system is overloaded whenever at least one resource is overloaded.
- * The class is used by the {@apilink AutoscaledPool} class.
+ * The class is used by the {@link AutoscaledPool} class.
  *
- * {@apilink SystemStatus.getCurrentStatus}
+ * {@link SystemStatus.getCurrentStatus}
  * returns a boolean that represents the current status of the system.
  * The length of the current timeframe in seconds is configurable
  * by the `currentHistorySecs` option and represents the max age
  * of snapshots to be considered for the calculation.
  *
- * {@apilink SystemStatus.getHistoricalStatus}
+ * {@link SystemStatus.getHistoricalStatus}
  * returns a boolean that represents the long-term status
  * of the system. It considers the full snapshot history available
- * in the {@apilink Snapshotter} instance.
+ * in the {@link Snapshotter} instance.
  * @category Scaling
  */
 export declare class SystemStatus {
@@ -112,7 +112,7 @@ export declare class SystemStatus {
     private readonly snapshotter;
     constructor(options?: SystemStatusOptions);
     /**
-     * Returns an {@apilink SystemInfo} object with the following structure:
+     * Returns an {@link SystemInfo} object with the following structure:
      *
      * ```javascript
      * {
@@ -129,7 +129,7 @@ export declare class SystemStatus {
      */
     getCurrentStatus(): SystemInfo;
     /**
-     * Returns an {@apilink SystemInfo} object with the following structure:
+     * Returns an {@link SystemInfo} object with the following structure:
      *
      * ```javascript
      * {
@@ -141,8 +141,8 @@ export declare class SystemStatus {
      * ```
      *
      * Where the `isSystemIdle` property is set to `false` if the system
-     * has been overloaded in the full history of the {@apilink Snapshotter}
-     * (which is configurable in the {@apilink Snapshotter}) and `true` otherwise.
+     * has been overloaded in the full history of the {@link Snapshotter}
+     * (which is configurable in the {@link Snapshotter}) and `true` otherwise.
      */
     getHistoricalStatus(): SystemInfo;
     /**

package/autoscaling/system_status.js

@@ -6,25 +6,25 @@ const utils_1 = require("@crawlee/utils");
 const ow_1 = tslib_1.__importDefault(require("ow"));
 const snapshotter_1 = require("./snapshotter");
 /**
- * Provides a simple interface to reading system status from a {@apilink Snapshotter} instance.
- * It only exposes two functions {@apilink SystemStatus.getCurrentStatus}
- * and {@apilink SystemStatus.getHistoricalStatus}.
+ * Provides a simple interface to reading system status from a {@link Snapshotter} instance.
+ * It only exposes two functions {@link SystemStatus.getCurrentStatus}
+ * and {@link SystemStatus.getHistoricalStatus}.
  * The system status is calculated using a weighted average of overloaded
  * messages in the snapshots, with the weights being the time intervals
  * between the snapshots. Each resource is calculated separately
  * and the system is overloaded whenever at least one resource is overloaded.
- * The class is used by the {@apilink AutoscaledPool} class.
+ * The class is used by the {@link AutoscaledPool} class.
  *
- * {@apilink SystemStatus.getCurrentStatus}
+ * {@link SystemStatus.getCurrentStatus}
  * returns a boolean that represents the current status of the system.
  * The length of the current timeframe in seconds is configurable
  * by the `currentHistorySecs` option and represents the max age
  * of snapshots to be considered for the calculation.
  *
- * {@apilink SystemStatus.getHistoricalStatus}
+ * {@link SystemStatus.getHistoricalStatus}
  * returns a boolean that represents the long-term status
  * of the system. It considers the full snapshot history available
- * in the {@apilink Snapshotter} instance.
+ * in the {@link Snapshotter} instance.
  * @category Scaling
  */
 class SystemStatus {
@@ -83,7 +83,7 @@ class SystemStatus {
         this.snapshotter = snapshotter || new snapshotter_1.Snapshotter({ config });
     }
     /**
-     * Returns an {@apilink SystemInfo} object with the following structure:
+     * Returns an {@link SystemInfo} object with the following structure:
      *
      * ```javascript
      * {
@@ -102,7 +102,7 @@ class SystemStatus {
         return this._isSystemIdle(this.currentHistoryMillis);
     }
     /**
-     * Returns an {@apilink SystemInfo} object with the following structure:
+     * Returns an {@link SystemInfo} object with the following structure:
      *
      * ```javascript
      * {
@@ -114,8 +114,8 @@ class SystemStatus {
      * ```
      *
      * Where the `isSystemIdle` property is set to `false` if the system
-     * has been overloaded in the full history of the {@apilink Snapshotter}
-     * (which is configurable in the {@apilink Snapshotter}) and `true` otherwise.
+     * has been overloaded in the full history of the {@link Snapshotter}
+     * (which is configurable in the {@link Snapshotter}) and `true` otherwise.
      */
     getHistoricalStatus() {
         return this._isSystemIdle();

package/configuration.d.ts

@@ -9,17 +9,17 @@ import { type Constructor } from './typedefs';
 export interface ConfigurationOptions {
     /**
      * Defines storage client to be used.
-     * @default {@apilink MemoryStorage}
+     * @default {@link MemoryStorage}
      */
     storageClient?: StorageClient;
     /**
      * Defines the Event Manager to be used.
-     * @default {@apilink EventManager}
+     * @default {@link EventManager}
      */
     eventManager?: EventManager;
     /**
      * Could be used to adjust the storage client behavior
-     * e.g. {@apilink MemoryStorageOptions} could be used to adjust the {@apilink MemoryStorage} behavior.
+     * e.g. {@link MemoryStorageOptions} could be used to adjust the {@link MemoryStorage} behavior.
      */
     storageClientOptions?: Dictionary;
     /**
@@ -57,7 +57,7 @@ export interface ConfigurationOptions {
      */
     maxUsedCpuRatio?: number;
     /**
-     * Sets the ratio, defining the amount of system memory that could be used by the {@apilink AutoscaledPool}.
+     * Sets the ratio, defining the amount of system memory that could be used by the {@link AutoscaledPool}.
      * When the memory usage is more than the provided ratio, the memory is considered overloaded.
      *
      * Alternative to `CRAWLEE_AVAILABLE_MEMORY_RATIO` environment variable.
@@ -65,7 +65,7 @@ export interface ConfigurationOptions {
      */
     availableMemoryRatio?: number;
     /**
-     * Sets the amount of system memory in megabytes to be used by the {@apilink AutoscaledPool}.
+     * Sets the amount of system memory in megabytes to be used by the {@link AutoscaledPool}.
      * By default, the maximum memory is set to one quarter of total system memory.
      *
      * Alternative to `CRAWLEE_MEMORY_MBYTES` environment variable.
@@ -85,7 +85,7 @@ export interface ConfigurationOptions {
     systemInfoIntervalMillis?: number;
     /**
      * Defines the default input key, i.e. the key that is used to get the crawler input value
-     * from the default {@apilink KeyValueStore} associated with the current crawler run.
+     * from the default {@link KeyValueStore} associated with the current crawler run.
      *
      * Alternative to `CRAWLEE_INPUT_KEY` environment variable.
      * @default 'INPUT'
@@ -257,8 +257,8 @@ export declare class Configuration {
      */
     static set(key: keyof ConfigurationOptions, value?: any): void;
     /**
-     * Returns cached instance of {@apilink StorageClient} using options as defined in the environment variables or in
-     * this {@apilink Configuration} instance. Only first call of this method will create the client, following calls will
+     * Returns cached instance of {@link StorageClient} using options as defined in the environment variables or in
+     * this {@link Configuration} instance. Only first call of this method will create the client, following calls will
      * return the same client instance.
      *
      * Caching works based on the `storageClientOptions`, so calling this method with different options will return
@@ -280,11 +280,11 @@ export declare class Configuration {
      */
     static getGlobalConfig(): Configuration;
     /**
-     * Gets default {@apilink StorageClient} instance.
+     * Gets default {@link StorageClient} instance.
      */
     static getStorageClient(): StorageClient;
     /**
-     * Gets default {@apilink EventManager} instance.
+     * Gets default {@link EventManager} instance.
      */
     static getEventManager(): EventManager;
     /**

package/configuration.js

@@ -169,8 +169,8 @@ class Configuration {
         this.getGlobalConfig().set(key, value);
     }
     /**
-     * Returns cached instance of {@apilink StorageClient} using options as defined in the environment variables or in
-     * this {@apilink Configuration} instance. Only first call of this method will create the client, following calls will
+     * Returns cached instance of {@link StorageClient} using options as defined in the environment variables or in
+     * this {@link Configuration} instance. Only first call of this method will create the client, following calls will
      * return the same client instance.
      *
      * Caching works based on the `storageClientOptions`, so calling this method with different options will return
@@ -232,13 +232,13 @@ class Configuration {
         return Configuration.globalConfig;
     }
     /**
-     * Gets default {@apilink StorageClient} instance.
+     * Gets default {@link StorageClient} instance.
      */
     static getStorageClient() {
         return this.getGlobalConfig().getStorageClient();
     }
     /**
-     * Gets default {@apilink EventManager} instance.
+     * Gets default {@link EventManager} instance.
      */
     static getEventManager() {
         return this.getGlobalConfig().getEventManager();

package/crawlers/crawler_commons.d.ts

@@ -26,15 +26,15 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
     session?: Session;
     /**
      * An object with information about currently used proxy by the crawler
-     * and configured by the {@apilink ProxyConfiguration} class.
+     * and configured by the {@link ProxyConfiguration} class.
      */
     proxyInfo?: ProxyInfo;
     /**
-     * The original {@apilink Request} object.
+     * The original {@link Request} object.
      */
     request: Request<UserData>;
     /**
-     * This function allows you to push data to a {@apilink Dataset} specified by name, or the one currently used by the crawler.
+     * This function allows you to push data to a {@link Dataset} specified by name, or the one currently used by the crawler.
      *
      * Shortcut for `crawler.pushData()`.
      *
@@ -42,11 +42,11 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
      */
     pushData(data: ReadonlyDeep<Parameters<Dataset['pushData']>[0]>, datasetIdOrName?: string): Promise<void>;
     /**
-     * This function automatically finds and enqueues links from the current page, adding them to the {@apilink RequestQueue}
+     * This function automatically finds and enqueues links from the current page, adding them to the {@link RequestQueue}
      * currently used by the crawler.
      *
      * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
-     * and override settings of the enqueued {@apilink Request} objects.
+     * and override settings of the enqueued {@link Request} objects.
      *
      * Check out the [Crawl a website with relative links](https://crawlee.dev/js/docs/examples/crawl-relative-links) example
      * for more details regarding its usage.
@@ -89,11 +89,11 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
 export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary = Dictionary> extends RestrictedCrawlingContext<UserData> {
     crawler: Crawler;
     /**
-     * This function automatically finds and enqueues links from the current page, adding them to the {@apilink RequestQueue}
+     * This function automatically finds and enqueues links from the current page, adding them to the {@link RequestQueue}
      * currently used by the crawler.
      *
      * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
-     * and override settings of the enqueued {@apilink Request} objects.
+     * and override settings of the enqueued {@link Request} objects.
      *
      * Check out the [Crawl a website with relative links](https://crawlee.dev/js/docs/examples/crawl-relative-links) example
      * for more details regarding its usage.
@@ -111,7 +111,7 @@ export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary
      * ```
      *
      * @param [options] All `enqueueLinks()` parameters are passed via an options object.
-     * @returns Promise that resolves to {@apilink BatchAddRequestsResult} object.
+     * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
      */
     enqueueLinks(options?: ReadonlyDeep<Omit<EnqueueLinksOptions, 'requestQueue'>> & Pick<EnqueueLinksOptions, 'requestQueue'>): Promise<BatchAddRequestsResult>;
     /**
@@ -138,7 +138,7 @@ export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary
     sendRequest<Response = string>(overrideOptions?: Partial<OptionsInit>): Promise<GotResponse<Response>>;
 }
 /**
- * A partial implementation of {@apilink RestrictedCrawlingContext} that stores parameters of calls to context methods for later inspection.
+ * A partial implementation of {@link RestrictedCrawlingContext} that stores parameters of calls to context methods for later inspection.
  *
  * @experimental
  */
@@ -151,7 +151,7 @@ export declare class RequestHandlerResult {
     private enqueueLinksCalls;
     constructor(config: Configuration, crawleeStateKey: string);
     /**
-     * A record of calls to {@apilink RestrictedCrawlingContext.pushData}, {@apilink RestrictedCrawlingContext.addRequests}, {@apilink RestrictedCrawlingContext.enqueueLinks} made by a request handler.
+     * A record of calls to {@link RestrictedCrawlingContext.pushData}, {@link RestrictedCrawlingContext.addRequests}, {@link RestrictedCrawlingContext.enqueueLinks} made by a request handler.
      */
     get calls(): ReadonlyDeep<{
         pushData: Parameters<RestrictedCrawlingContext['pushData']>[];
@@ -173,14 +173,14 @@ export declare class RequestHandlerResult {
         datasetIdOrName?: string;
     }[]>;
     /**
-     * URLs enqueued to the request queue by a request handler, either via {@apilink RestrictedCrawlingContext.addRequests} or {@apilink RestrictedCrawlingContext.enqueueLinks}
+     * URLs enqueued to the request queue by a request handler, either via {@link RestrictedCrawlingContext.addRequests} or {@link RestrictedCrawlingContext.enqueueLinks}
      */
     get enqueuedUrls(): ReadonlyDeep<{
         url: string;
         label?: string;
     }[]>;
     /**
-     * URL lists enqueued to the request queue by a request handler via {@apilink RestrictedCrawlingContext.addRequests} using the `requestsFromUrl` option.
+     * URL lists enqueued to the request queue by a request handler via {@link RestrictedCrawlingContext.addRequests} using the `requestsFromUrl` option.
      */
     get enqueuedUrlLists(): ReadonlyDeep<{
         listUrl: string;

package/crawlers/crawler_commons.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"crawler_commons.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEzE,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,IAAI,WAAW,EAAE,MAAM,cAAc,CAAC;AACzE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAE9C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,gCAAgC,CAAC;AAC1E,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAClD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,4BAA4B,EAAE,MAAM,aAAa,CAAC;AACxF,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,gBAAgB;AAChB,MAAM,MAAM,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,KAAK,CAAC;AAEtD,gBAAgB;AAChB,MAAM,MAAM,YAAY,CAAC,CAAC,EAAE,CAAC,SAAS,MAAM,CAAC,IAAI,CAAC,GAAG;KAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;CAAE,CAAC;AAE1E,MAAM,MAAM,aAAa,CAAC,CAAC,SAAS,OAAO,IAAI,YAAY,CAAC,CAAC,EAAE,IAAI,GAAG,WAAW,CAAC,CAAC;AAEnF,gBAAgB;AAChB,MAAM,MAAM,aAAa,CAAC,OAAO,SAAS,yBAAyB,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,IAAI,GAC5F,OAAO,GACP;IACI,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;CAC9C,GAAG,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;AAEnC,MAAM,WAAW,yBAAyB,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU,CAE/E,SAAQ,MAAM,CAAC,MAAM,GAAG,EAAE,EAAE,OAAO,CAAC;IACpC,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IAEtB;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAE3B;;;;;;OAMG;IACH,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC,UAAU,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,eAAe,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE1G;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,YAAY,EAAE,CAAC,OAAO,CAAC,EAAE,YAAY,CAAC,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAEtG;;;;;OAKG;IACH,WAAW,EAAE,CACT,YAAY,EAAE,YAAY,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC,EAC/C,OAAO,CAAC,EAAE,YAAY,CAAC,4BAA4B,CAAC,KACnD,OAAO,CAAC,IAAI,CAAC,CAAC;IAEnB;;OAEG;IACH,QAAQ,EAAE,CAAC,KAAK,SAAS,UAAU,GAAG,UAAU,EAAE,YAAY,CAAC,EAAE,KAAK,KAAK,OAAO,CAAC,KAAK,CAAC,CAAC;IAE1F;;OAEG;IACH,gBAAgB,EAAE,CACd,QAAQ,CAAC,EAAE,MAAM,KAChB,OAAO,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,GAAG,MAAM,GAAG,UAAU,GAAG,mBAAmB,GAAG,UAAU,GAAG,cAAc,CAAC,CAAC,CAAC;IAElH;;OAEG;IACH,GAAG,EAAE,GAAG,CAAC;CACZ;AAED,MAAM,WAAW,eAAe,CAAC,OAAO,GAAG,OAAO,EAAE,QAAQ,SAAS,UAAU,GAAG,UAAU,CACxF,SAAQ,yBAAyB,CAAC,QAAQ,CAAC;IAC3C,OAAO,EAAE,OAAO,CAAC;IAEjB;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,YAAY,CACR,OAAO,CAAC,EAAE,YAAY,CAAC,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC,GAAG,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,GAC9G,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAEnC;;OAEG;IACH,gBAAgB,EAAE,CAAC,QAAQ,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,aAAa,CAAC,CAAC;IAEhE;;;;;;;;;;;;;;;;OAgBG;IACH,WAAW,CAAC,QAAQ,GAAG,MAAM,EAAE,eAAe,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC;CAC1G;AAED;;;;GAIG;AACH,qBAAa,oBAAoB;IAWzB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,eAAe;IAX3B,OAAO,CAAC,qBAAqB,CACtB;IAEP,OAAO,CAAC,aAAa,CAA2D;IAEhF,OAAO,CAAC,gBAAgB,CAA8D;IAEtF,OAAO,CAAC,iBAAiB,CAA+D;gBAG5E,MAAM,EAAE,aAAa,EACrB,eAAe,EAAE,MAAM;IAGnC;;OAEG;IACH,IAAI,KAAK,IAAI,YAAY,CAAC;QACtB,QAAQ,EAAE,UAAU,CAAC,yBAAyB,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;QAC9D,WAAW,EAAE,UAAU,CAAC,yBAAyB,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC;QACpE,YAAY,EAAE,UAAU,CAAC,yBAAyB,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC;KACzE,CAAC,CAMD;IAED;;OAEG;IACH,IAAI,oBAAoB,IAAI,YAAY,CACpC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,YAAY,EAAE,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,aAAa,CAAA;KAAE,CAAC,CAAC,CACrF,CAEA;IAED;;OAEG;IACH,IAAI,YAAY,IAAI,YAAY,CAAC;QAAE,IAAI,EAAE,UAAU,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAIjF;IAED;;OAEG;IACH,IAAI,YAAY,IAAI,YAAY,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAsBlE;IAED;;OAEG;IACH,IAAI,gBAAgB,IAAI,YAAY,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAgB1E;IAED,QAAQ,EAAE,yBAAyB,CAAC,UAAU,CAAC,CAE7C;IAEF,YAAY,EAAE,yBAAyB,CAAC,cAAc,CAAC,CAErD;IAEF,WAAW,EAAE,yBAAyB,CAAC,aAAa,CAAC,CAEnD;IAEF,QAAQ,EAAE,yBAAyB,CAAC,UAAU,CAAC,CAG7C;IAEF,gBAAgB,EAAE,yBAAyB,CAAC,kBAAkB,CAAC,CAqB7D;IAEF,OAAO,CAAC,WAAW,CAAwF;IAE3G,OAAO,CAAC,4BAA4B,CAIlC;IAEF,OAAO,CAAC,4BAA4B,CASlC;CACL"}
+{"version":3,"file":"crawler_commons.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEzE,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,IAAI,WAAW,EAAE,MAAM,cAAc,CAAC;AACzE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAE9C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,gCAAgC,CAAC;AAC1E,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAClD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,4BAA4B,EAAE,MAAM,aAAa,CAAC;AACxF,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,gBAAgB;AAChB,MAAM,MAAM,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,KAAK,CAAC;AAEtD,gBAAgB;AAChB,MAAM,MAAM,YAAY,CAAC,CAAC,EAAE,CAAC,SAAS,MAAM,CAAC,IAAI,CAAC,GAAG;KAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;CAAE,CAAC;AAE1E,MAAM,MAAM,aAAa,CAAC,CAAC,SAAS,OAAO,IAAI,YAAY,CAAC,CAAC,EAAE,IAAI,GAAG,WAAW,CAAC,CAAC;AAEnF,gBAAgB;AAChB,MAAM,MAAM,aAAa,CAAC,OAAO,SAAS,yBAAyB,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,IAAI,GAC5F,OAAO,GACP;IACI,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;CAC9C,GAAG,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;AAEnC,MAAM,WAAW,yBAAyB,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU,CAE/E,SAAQ,MAAM,CAAC,MAAM,GAAG,EAAE,EAAE,OAAO,CAAC;IACpC,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IAEtB;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAE3B;;;;;;OAMG;IACH,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC,UAAU,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,eAAe,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE1G;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,YAAY,EAAE,CAAC,OAAO,CAAC,EAAE,YAAY,CAAC,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAEtG;;;;;OAKG;IACH,WAAW,EAAE,CACT,YAAY,EAAE,YAAY,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC,EAC/C,OAAO,CAAC,EAAE,YAAY,CAAC,4BAA4B,CAAC,KACnD,OAAO,CAAC,IAAI,CAAC,CAAC;IAEnB;;OAEG;IACH,QAAQ,EAAE,CAAC,KAAK,SAAS,UAAU,GAAG,UAAU,EAAE,YAAY,CAAC,EAAE,KAAK,KAAK,OAAO,CAAC,KAAK,CAAC,CAAC;IAE1F;;OAEG;IACH,gBAAgB,EAAE,CACd,QAAQ,CAAC,EAAE,MAAM,KAChB,OAAO,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,GAAG,MAAM,GAAG,UAAU,GAAG,mBAAmB,GAAG,UAAU,GAAG,cAAc,CAAC,CAAC,CAAC;IAElH;;OAEG;IACH,GAAG,EAAE,GAAG,CAAC;CACZ;AAED,MAAM,WAAW,eAAe,CAAC,OAAO,GAAG,OAAO,EAAE,QAAQ,SAAS,UAAU,GAAG,UAAU,CACxF,SAAQ,yBAAyB,CAAC,QAAQ,CAAC;IAC3C,OAAO,EAAE,OAAO,CAAC;IAEjB;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,YAAY,CACR,OAAO,CAAC,EAAE,YAAY,CAAC,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC,GAAG,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,GAC9G,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAEnC;;OAEG;IACH,gBAAgB,EAAE,CAAC,QAAQ,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,aAAa,CAAC,CAAC;IAEhE;;;;;;;;;;;;;;;;OAgBG;IACH,WAAW,CAAC,QAAQ,GAAG,MAAM,EAAE,eAAe,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC;CAC1G;AAED;;;;GAIG;AACH,qBAAa,oBAAoB;IAWzB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,eAAe;IAX3B,OAAO,CAAC,qBAAqB,CACtB;IAEP,OAAO,CAAC,aAAa,CAA2D;IAEhF,OAAO,CAAC,gBAAgB,CAA8D;IAEtF,OAAO,CAAC,iBAAiB,CAA+D;gBAG5E,MAAM,EAAE,aAAa,EACrB,eAAe,EAAE,MAAM;IAGnC;;OAEG;IACH,IAAI,KAAK,IAAI,YAAY,CAAC;QACtB,QAAQ,EAAE,UAAU,CAAC,yBAAyB,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;QAC9D,WAAW,EAAE,UAAU,CAAC,yBAAyB,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC;QACpE,YAAY,EAAE,UAAU,CAAC,yBAAyB,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC;KACzE,CAAC,CAMD;IAED;;OAEG;IACH,IAAI,oBAAoB,IAAI,YAAY,CACpC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,YAAY,EAAE,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,aAAa,CAAA;KAAE,CAAC,CAAC,CACrF,CAEA;IAED;;OAEG;IACH,IAAI,YAAY,IAAI,YAAY,CAAC;QAAE,IAAI,EAAE,UAAU,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAIjF;IAED;;OAEG;IACH,IAAI,YAAY,IAAI,YAAY,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAsBlE;IAED;;OAEG;IACH,IAAI,gBAAgB,IAAI,YAAY,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAgB1E;IAED,QAAQ,EAAE,yBAAyB,CAAC,UAAU,CAAC,CAE7C;IAEF,YAAY,EAAE,yBAAyB,CAAC,cAAc,CAAC,CAErD;IAEF,WAAW,EAAE,yBAAyB,CAAC,aAAa,CAAC,CAEnD;IAEF,QAAQ,EAAE,yBAAyB,CAAC,UAAU,CAAC,CAG7C;IAEF,gBAAgB,EAAE,yBAAyB,CAAC,kBAAkB,CAAC,CAa7D;IAEF,OAAO,CAAC,WAAW,CAAwF;IAE3G,OAAO,CAAC,4BAA4B,CAIlC;IAEF,OAAO,CAAC,4BAA4B,CASlC;CACL"}

package/crawlers/crawler_commons.js

@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.RequestHandlerResult = void 0;
 const storages_1 = require("../storages");
 /**
- * A partial implementation of {@apilink RestrictedCrawlingContext} that stores parameters of calls to context methods for later inspection.
+ * A partial implementation of {@link RestrictedCrawlingContext} that stores parameters of calls to context methods for later inspection.
  *
  * @experimental
  */
@@ -88,17 +88,10 @@ class RequestHandlerResult {
                     id: this.idOrDefault(idOrName),
                     name: idOrName,
                     getValue: async (key) => this.getKeyValueStoreChangedValue(idOrName, key) ?? (await store.getValue(key)),
-                    getAutoSavedValue: async (key, defaultValue = {}) => {
-                        let value = this.getKeyValueStoreChangedValue(idOrName, key);
-                        if (value === null) {
-                            value = (await store.getValue(key)) ?? defaultValue;
-                            this.setKeyValueStoreChangedValue(idOrName, key, value);
-                        }
-                        return value;
-                    },
                     setValue: async (key, value, options) => {
                         this.setKeyValueStoreChangedValue(idOrName, key, value, options);
                     },
+                    getAutoSavedValue: store.getAutoSavedValue.bind(store),
                     getPublicUrl: store.getPublicUrl.bind(store),
                 };
             }
@@ -133,7 +126,7 @@ class RequestHandlerResult {
         });
     }
     /**
-     * A record of calls to {@apilink RestrictedCrawlingContext.pushData}, {@apilink RestrictedCrawlingContext.addRequests}, {@apilink RestrictedCrawlingContext.enqueueLinks} made by a request handler.
+     * A record of calls to {@link RestrictedCrawlingContext.pushData}, {@link RestrictedCrawlingContext.addRequests}, {@link RestrictedCrawlingContext.enqueueLinks} made by a request handler.
      */
     get calls() {
         return {
@@ -155,7 +148,7 @@ class RequestHandlerResult {
         return this.pushDataCalls.flatMap(([data, datasetIdOrName]) => (Array.isArray(data) ? data : [data]).map((item) => ({ item, datasetIdOrName })));
     }
     /**
-     * URLs enqueued to the request queue by a request handler, either via {@apilink RestrictedCrawlingContext.addRequests} or {@apilink RestrictedCrawlingContext.enqueueLinks}
+     * URLs enqueued to the request queue by a request handler, either via {@link RestrictedCrawlingContext.addRequests} or {@link RestrictedCrawlingContext.enqueueLinks}
      */
     get enqueuedUrls() {
         const result = [];
@@ -177,7 +170,7 @@ class RequestHandlerResult {
         return result;
     }
     /**
-     * URL lists enqueued to the request queue by a request handler via {@apilink RestrictedCrawlingContext.addRequests} using the `requestsFromUrl` option.
+     * URL lists enqueued to the request queue by a request handler via {@link RestrictedCrawlingContext.addRequests} using the `requestsFromUrl` option.
      */
     get enqueuedUrlLists() {
         const result = [];

package/crawlers/crawler_commons.js.map

@@ -1 +1 @@
-{"version":3,"file":"crawler_commons.js","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":";;;AAYA,0CAA4C;AA4J5C;;;;GAIG;AACH,MAAa,oBAAoB;IAU7B,YACY,MAAqB,EACrB,eAAuB;QAD/B;;;;mBAAQ,MAAM;WAAe;QAC7B;;;;mBAAQ,eAAe;WAAQ;QAX3B;;;;mBACJ,EAAE;WAAC;QAEC;;;;mBAAqE,EAAE;WAAC;QAExE;;;;mBAA2E,EAAE;WAAC;QAE9E;;;;mBAA6E,EAAE;WAAC;QAwFxF;;;;mBAAkD,KAAK,EAAE,IAAI,EAAE,eAAe,EAAE,EAAE;gBAC9E,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC,CAAC;YACrD,CAAC;WAAC;QAEF;;;;mBAA0D,KAAK,EAAE,OAAO,EAAE,EAAE;gBACxE,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YAC3C,CAAC;WAAC;QAEF;;;;mBAAwD,KAAK,EAAE,QAAQ,EAAE,OAAO,GAAG,EAAE,EAAE,EAAE;gBACrF,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;YACpD,CAAC;WAAC;QAEF;;;;mBAAkD,KAAK,EAAE,YAAY,EAAE,EAAE;gBACrE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;gBACrD,OAAO,MAAM,KAAK,CAAC,iBAAiB,CAAC,IAAI,CAAC,eAAe,EAAE,YAAY,CAAC,CAAC;YAC7E,CAAC;WAAC;QAEF;;;;mBAAkE,KAAK,EAAE,QAAQ,EAAE,EAAE;gBACjF,MAAM,KAAK,GAAG,MAAM,wBAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;gBAE1E,OAAO;oBACH,EAAE,EAAE,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC;oBAC9B,IAAI,EAAE,QAAQ;oBACd,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,4BAA4B,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;oBACxG,iBAAiB,EAAE,KAAK,EAAqC,GAAW,EAAE,eAAkB,EAAO,EAAE,EAAE;wBACnG,IAAI,KAAK,GAAG,IAAI,CAAC,4BAA4B,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;wBAC7D,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;4BACjB,KAAK,GAAG,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,YAAY,CAAC;4BACpD,IAAI,CAAC,4BAA4B,CAAC,QAAQ,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;wBAC5D,CAAC;wBAED,OAAO,KAAU,CAAC;oBACtB,CAAC;oBACD,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;wBACpC,IAAI,CAAC,4BAA4B,CAAC,QAAQ,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;oBACrE,CAAC;oBACD,YAAY,EAAE,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC;iBAC/C,CAAC;YACN,CAAC;WAAC;QAEM;;;;mBAAc,CAAC,QAAiB,EAAU,EAAE,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,wBAAwB,CAAC;WAAC;QAEnG;;;;mBAA+B,CAAC,QAA4B,EAAE,GAAW,EAAE,EAAE;;gBACjF,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBACtC,MAAA,IAAI,CAAC,qBAAqB,EAAC,EAAE,SAAF,EAAE,IAAM,EAAE,EAAC;gBACtC,OAAO,IAAI,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,YAAY,IAAI,IAAI,CAAC;YACpE,CAAC;WAAC;QAEM;;;;mBAA+B,CACnC,QAA4B,EAC5B,GAAW,EACX,YAAqB,EACrB,OAAuB,EACzB,EAAE;;gBACA,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBACtC,MAAA,IAAI,CAAC,qBAAqB,EAAC,EAAE,SAAF,EAAE,IAAM,EAAE,EAAC;gBACtC,IAAI,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC;YACpE,CAAC;WAAC;IA5IC,CAAC;IAEJ;;OAEG;IACH,IAAI,KAAK;QAKL,OAAO;YACH,QAAQ,EAAE,IAAI,CAAC,aAAa;YAC5B,WAAW,EAAE,IAAI,CAAC,gBAAgB;YAClC,YAAY,EAAE,IAAI,CAAC,iBAAiB;SACvC,CAAC;IACN,CAAC;IAED;;OAEG;IACH,IAAI,oBAAoB;QAGpB,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,IAAI,YAAY;QACZ,OAAO,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,eAAe,CAAC,EAAE,EAAE,CAC1D,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC,CACnF,CAAC;IACN,CAAC;IAED;;OAEG;IACH,IAAI,YAAY;QACZ,MAAM,MAAM,GAAsC,EAAE,CAAC;QAErD,KAAK,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC1F,CAAC;QAED,KAAK,MAAM,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC7C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC7B,IACI,OAAO,OAAO,KAAK,QAAQ;oBAC3B,CAAC,CAAC,CAAC,iBAAiB,IAAI,OAAO,CAAC,IAAI,OAAO,CAAC,eAAe,KAAK,SAAS,CAAC;oBAC1E,OAAO,CAAC,GAAG,KAAK,SAAS,EAC3B,CAAC;oBACC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;gBAC5D,CAAC;qBAAM,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;oBACrC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;gBAClC,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,IAAI,gBAAgB;QAChB,MAAM,MAAM,GAA0C,EAAE,CAAC;QAEzD,KAAK,MAAM,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC7C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC7B,IACI,OAAO,OAAO,KAAK,QAAQ;oBAC3B,iBAAiB,IAAI,OAAO;oBAC5B,OAAO,CAAC,eAAe,KAAK,SAAS,EACvC,CAAC;oBACC,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,eAAe,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;gBAC5E,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;CA4DJ;AA1JD,oDA0JC"}
+{"version":3,"file":"crawler_commons.js","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":";;;AAYA,0CAA4C;AA4J5C;;;;GAIG;AACH,MAAa,oBAAoB;IAU7B,YACY,MAAqB,EACrB,eAAuB;QAD/B;;;;mBAAQ,MAAM;WAAe;QAC7B;;;;mBAAQ,eAAe;WAAQ;QAX3B;;;;mBACJ,EAAE;WAAC;QAEC;;;;mBAAqE,EAAE;WAAC;QAExE;;;;mBAA2E,EAAE;WAAC;QAE9E;;;;mBAA6E,EAAE;WAAC;QAwFxF;;;;mBAAkD,KAAK,EAAE,IAAI,EAAE,eAAe,EAAE,EAAE;gBAC9E,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC,CAAC;YACrD,CAAC;WAAC;QAEF;;;;mBAA0D,KAAK,EAAE,OAAO,EAAE,EAAE;gBACxE,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YAC3C,CAAC;WAAC;QAEF;;;;mBAAwD,KAAK,EAAE,QAAQ,EAAE,OAAO,GAAG,EAAE,EAAE,EAAE;gBACrF,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;YACpD,CAAC;WAAC;QAEF;;;;mBAAkD,KAAK,EAAE,YAAY,EAAE,EAAE;gBACrE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;gBACrD,OAAO,MAAM,KAAK,CAAC,iBAAiB,CAAC,IAAI,CAAC,eAAe,EAAE,YAAY,CAAC,CAAC;YAC7E,CAAC;WAAC;QAEF;;;;mBAAkE,KAAK,EAAE,QAAQ,EAAE,EAAE;gBACjF,MAAM,KAAK,GAAG,MAAM,wBAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;gBAE1E,OAAO;oBACH,EAAE,EAAE,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC;oBAC9B,IAAI,EAAE,QAAQ;oBACd,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,4BAA4B,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;oBACxG,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;wBACpC,IAAI,CAAC,4BAA4B,CAAC,QAAQ,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;oBACrE,CAAC;oBACD,iBAAiB,EAAE,KAAK,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC;oBACtD,YAAY,EAAE,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC;iBAC/C,CAAC;YACN,CAAC;WAAC;QAEM;;;;mBAAc,CAAC,QAAiB,EAAU,EAAE,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,wBAAwB,CAAC;WAAC;QAEnG;;;;mBAA+B,CAAC,QAA4B,EAAE,GAAW,EAAE,EAAE;;gBACjF,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBACtC,MAAA,IAAI,CAAC,qBAAqB,EAAC,EAAE,SAAF,EAAE,IAAM,EAAE,EAAC;gBACtC,OAAO,IAAI,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,YAAY,IAAI,IAAI,CAAC;YACpE,CAAC;WAAC;QAEM;;;;mBAA+B,CACnC,QAA4B,EAC5B,GAAW,EACX,YAAqB,EACrB,OAAuB,EACzB,EAAE;;gBACA,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBACtC,MAAA,IAAI,CAAC,qBAAqB,EAAC,EAAE,SAAF,EAAE,IAAM,EAAE,EAAC;gBACtC,IAAI,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC;YACpE,CAAC;WAAC;IApIC,CAAC;IAEJ;;OAEG;IACH,IAAI,KAAK;QAKL,OAAO;YACH,QAAQ,EAAE,IAAI,CAAC,aAAa;YAC5B,WAAW,EAAE,IAAI,CAAC,gBAAgB;YAClC,YAAY,EAAE,IAAI,CAAC,iBAAiB;SACvC,CAAC;IACN,CAAC;IAED;;OAEG;IACH,IAAI,oBAAoB;QAGpB,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,IAAI,YAAY;QACZ,OAAO,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,eAAe,CAAC,EAAE,EAAE,CAC1D,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC,CACnF,CAAC;IACN,CAAC;IAED;;OAEG;IACH,IAAI,YAAY;QACZ,MAAM,MAAM,GAAsC,EAAE,CAAC;QAErD,KAAK,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC1F,CAAC;QAED,KAAK,MAAM,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC7C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC7B,IACI,OAAO,OAAO,KAAK,QAAQ;oBAC3B,CAAC,CAAC,CAAC,iBAAiB,IAAI,OAAO,CAAC,IAAI,OAAO,CAAC,eAAe,KAAK,SAAS,CAAC;oBAC1E,OAAO,CAAC,GAAG,KAAK,SAAS,EAC3B,CAAC;oBACC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;gBAC5D,CAAC;qBAAM,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;oBACrC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;gBAClC,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,IAAI,gBAAgB;QAChB,MAAM,MAAM,GAA0C,EAAE,CAAC;QAEzD,KAAK,MAAM,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC7C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC7B,IACI,OAAO,OAAO,KAAK,QAAQ;oBAC3B,iBAAiB,IAAI,OAAO;oBAC5B,OAAO,CAAC,eAAe,KAAK,SAAS,EACvC,CAAC;oBACC,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,eAAe,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;gBAC5E,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;CAoDJ;AAlJD,oDAkJC"}