@crawlee/core 3.0.3-beta.10 → 3.0.3-beta.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -20,7 +20,7 @@ export interface AutoscaledPoolOptions {
20
20
  * If it resolves to `true` then the pool's run finishes. Being called only
21
21
  * when there are no tasks being processed means that as long as `isTaskReadyFunction()`
22
22
  * keeps resolving to `true`, `isFinishedFunction()` will never be called.
23
- * To abort a run, use the {@link AutoscaledPool.abort} method.
23
+ * To abort a run, use the {@apilink AutoscaledPool.abort} method.
24
24
  */
25
25
  isFinishedFunction?: () => Promise<boolean>;
26
26
  /**
@@ -84,12 +84,12 @@ export interface AutoscaledPoolOptions {
84
84
  */
85
85
  taskTimeoutSecs?: number;
86
86
  /**
87
- * Options to be passed down to the {@link Snapshotter} constructor. This is useful for fine-tuning
87
+ * Options to be passed down to the {@apilink Snapshotter} constructor. This is useful for fine-tuning
88
88
  * the snapshot intervals and history.
89
89
  */
90
90
  snapshotterOptions?: SnapshotterOptions;
91
91
  /**
92
- * Options to be passed down to the {@link SystemStatus} constructor. This is useful for fine-tuning
92
+ * Options to be passed down to the {@apilink SystemStatus} constructor. This is useful for fine-tuning
93
93
  * the system status reports. If a custom snapshotter is set in the options, it will be used
94
94
  * by the pool.
95
95
  */
@@ -106,22 +106,22 @@ export interface AutoscaledPoolOptions {
106
106
  * The pool only starts new tasks if there is enough free CPU and memory available
107
107
  * and the Javascript event loop is not blocked.
108
108
  *
109
- * The information about the CPU and memory usage is obtained by the {@link Snapshotter} class,
109
+ * The information about the CPU and memory usage is obtained by the {@apilink Snapshotter} class,
110
110
  * which makes regular snapshots of system resources that may be either local
111
111
  * or from the Apify cloud infrastructure in case the process is running on the Apify platform.
112
- * Meaningful data gathered from these snapshots is provided to `AutoscaledPool` by the {@link SystemStatus} class.
112
+ * Meaningful data gathered from these snapshots is provided to `AutoscaledPool` by the {@apilink SystemStatus} class.
113
113
  *
114
114
  * Before running the pool, you need to implement the following three functions:
115
- * {@link AutoscaledPoolOptions.runTaskFunction},
116
- * {@link AutoscaledPoolOptions.isTaskReadyFunction} and
117
- * {@link AutoscaledPoolOptions.isFinishedFunction}.
115
+ * {@apilink AutoscaledPoolOptions.runTaskFunction},
116
+ * {@apilink AutoscaledPoolOptions.isTaskReadyFunction} and
117
+ * {@apilink AutoscaledPoolOptions.isFinishedFunction}.
118
118
  *
119
- * The auto-scaled pool is started by calling the {@link AutoscaledPool.run} function.
120
- * The pool periodically queries the {@link AutoscaledPoolOptions.isTaskReadyFunction} function
119
+ * The auto-scaled pool is started by calling the {@apilink AutoscaledPool.run} function.
120
+ * The pool periodically queries the {@apilink AutoscaledPoolOptions.isTaskReadyFunction} function
121
121
  * for more tasks, managing optimal concurrency, until the function resolves to `false`. The pool then queries
122
- * the {@link AutoscaledPoolOptions.isFinishedFunction}. If it resolves to `true`, the run finishes after all running tasks complete.
122
+ * the {@apilink AutoscaledPoolOptions.isFinishedFunction}. If it resolves to `true`, the run finishes after all running tasks complete.
123
123
  * If it resolves to `false`, it assumes there will be more tasks available later and keeps periodically querying for tasks.
124
- * If any of the tasks throws then the {@link AutoscaledPool.run} function rejects the promise with an error.
124
+ * If any of the tasks throws then the {@apilink AutoscaledPool.run} function rejects the promise with an error.
125
125
  *
126
126
  * The pool evaluates whether it should start a new task every time one of the tasks finishes
127
127
  * and also in the interval set by the `options.maybeRunIntervalSecs` parameter.
@@ -221,7 +221,7 @@ export declare class AutoscaledPool {
221
221
  run(): Promise<void>;
222
222
  /**
223
223
  * Aborts the run of the auto-scaled pool and destroys it. The promise returned from
224
- * the {@link AutoscaledPool.run} function will immediately resolve, no more new tasks
224
+ * the {@apilink AutoscaledPool.run} function will immediately resolve, no more new tasks
225
225
  * will be spawned and all running tasks will be left in their current state.
226
226
  *
227
227
  * Due to the nature of the tasks, auto-scaled pool cannot reliably guarantee abortion
@@ -233,19 +233,19 @@ export declare class AutoscaledPool {
233
233
  abort(): Promise<void>;
234
234
  /**
235
235
  * Prevents the auto-scaled pool from starting new tasks, but allows the running ones to finish
236
- * (unlike abort, which terminates them). Used together with {@link AutoscaledPool.resume}
236
+ * (unlike abort, which terminates them). Used together with {@apilink AutoscaledPool.resume}
237
237
  *
238
238
  * The function's promise will resolve once all running tasks have completed and the pool
239
239
  * is effectively idle. If the `timeoutSecs` argument is provided, the promise will reject
240
240
  * with a timeout error after the `timeoutSecs` seconds.
241
241
  *
242
- * The promise returned from the {@link AutoscaledPool.run} function will not resolve
242
+ * The promise returned from the {@apilink AutoscaledPool.run} function will not resolve
243
243
  * when `.pause()` is invoked (unlike abort, which resolves it).
244
244
  */
245
245
  pause(timeoutSecs?: number): Promise<void>;
246
246
  /**
247
247
  * Resumes the operation of the autoscaled-pool by allowing more tasks to be run.
248
- * Used together with {@link AutoscaledPool.pause}
248
+ * Used together with {@apilink AutoscaledPool.pause}
249
249
  *
250
250
  * Tasks will automatically start running again in `options.maybeRunIntervalSecs`.
251
251
  */
@@ -14,22 +14,22 @@ const system_status_1 = require("./system_status");
14
14
  * The pool only starts new tasks if there is enough free CPU and memory available
15
15
  * and the Javascript event loop is not blocked.
16
16
  *
17
- * The information about the CPU and memory usage is obtained by the {@link Snapshotter} class,
17
+ * The information about the CPU and memory usage is obtained by the {@apilink Snapshotter} class,
18
18
  * which makes regular snapshots of system resources that may be either local
19
19
  * or from the Apify cloud infrastructure in case the process is running on the Apify platform.
20
- * Meaningful data gathered from these snapshots is provided to `AutoscaledPool` by the {@link SystemStatus} class.
20
+ * Meaningful data gathered from these snapshots is provided to `AutoscaledPool` by the {@apilink SystemStatus} class.
21
21
  *
22
22
  * Before running the pool, you need to implement the following three functions:
23
- * {@link AutoscaledPoolOptions.runTaskFunction},
24
- * {@link AutoscaledPoolOptions.isTaskReadyFunction} and
25
- * {@link AutoscaledPoolOptions.isFinishedFunction}.
23
+ * {@apilink AutoscaledPoolOptions.runTaskFunction},
24
+ * {@apilink AutoscaledPoolOptions.isTaskReadyFunction} and
25
+ * {@apilink AutoscaledPoolOptions.isFinishedFunction}.
26
26
  *
27
- * The auto-scaled pool is started by calling the {@link AutoscaledPool.run} function.
28
- * The pool periodically queries the {@link AutoscaledPoolOptions.isTaskReadyFunction} function
27
+ * The auto-scaled pool is started by calling the {@apilink AutoscaledPool.run} function.
28
+ * The pool periodically queries the {@apilink AutoscaledPoolOptions.isTaskReadyFunction} function
29
29
  * for more tasks, managing optimal concurrency, until the function resolves to `false`. The pool then queries
30
- * the {@link AutoscaledPoolOptions.isFinishedFunction}. If it resolves to `true`, the run finishes after all running tasks complete.
30
+ * the {@apilink AutoscaledPoolOptions.isFinishedFunction}. If it resolves to `true`, the run finishes after all running tasks complete.
31
31
  * If it resolves to `false`, it assumes there will be more tasks available later and keeps periodically querying for tasks.
32
- * If any of the tasks throws then the {@link AutoscaledPool.run} function rejects the promise with an error.
32
+ * If any of the tasks throws then the {@apilink AutoscaledPool.run} function rejects the promise with an error.
33
33
  *
34
34
  * The pool evaluates whether it should start a new task every time one of the tasks finishes
35
35
  * and also in the interval set by the `options.maybeRunIntervalSecs` parameter.
@@ -378,7 +378,7 @@ class AutoscaledPool {
378
378
  }
379
379
  /**
380
380
  * Aborts the run of the auto-scaled pool and destroys it. The promise returned from
381
- * the {@link AutoscaledPool.run} function will immediately resolve, no more new tasks
381
+ * the {@apilink AutoscaledPool.run} function will immediately resolve, no more new tasks
382
382
  * will be spawned and all running tasks will be left in their current state.
383
383
  *
384
384
  * Due to the nature of the tasks, auto-scaled pool cannot reliably guarantee abortion
@@ -396,13 +396,13 @@ class AutoscaledPool {
396
396
  }
397
397
  /**
398
398
  * Prevents the auto-scaled pool from starting new tasks, but allows the running ones to finish
399
- * (unlike abort, which terminates them). Used together with {@link AutoscaledPool.resume}
399
+ * (unlike abort, which terminates them). Used together with {@apilink AutoscaledPool.resume}
400
400
  *
401
401
  * The function's promise will resolve once all running tasks have completed and the pool
402
402
  * is effectively idle. If the `timeoutSecs` argument is provided, the promise will reject
403
403
  * with a timeout error after the `timeoutSecs` seconds.
404
404
  *
405
- * The promise returned from the {@link AutoscaledPool.run} function will not resolve
405
+ * The promise returned from the {@apilink AutoscaledPool.run} function will not resolve
406
406
  * when `.pause()` is invoked (unlike abort, which resolves it).
407
407
  */
408
408
  async pause(timeoutSecs) {
@@ -431,7 +431,7 @@ class AutoscaledPool {
431
431
  }
432
432
  /**
433
433
  * Resumes the operation of the autoscaled-pool by allowing more tasks to be run.
434
- * Used together with {@link AutoscaledPool.pause}
434
+ * Used together with {@apilink AutoscaledPool.pause}
435
435
  *
436
436
  * Tasks will automatically start running again in `options.maybeRunIntervalSecs`.
437
437
  */
@@ -75,7 +75,7 @@ interface ClientSnapshot {
75
75
  * Creates snapshots of system resources at given intervals and marks the resource
76
76
  * as either overloaded or not during the last interval. Keeps a history of the snapshots.
77
77
  * It tracks the following resources: Memory, EventLoop, API and CPU.
78
- * The class is used by the {@link AutoscaledPool} class.
78
+ * The class is used by the {@apilink AutoscaledPool} class.
79
79
  *
80
80
  * When running on the Apify platform, the CPU and memory statistics are provided by the platform,
81
81
  * as collected from the running Docker container. When running locally, `Snapshotter`
@@ -14,7 +14,7 @@ const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10000;
14
14
  * Creates snapshots of system resources at given intervals and marks the resource
15
15
  * as either overloaded or not during the last interval. Keeps a history of the snapshots.
16
16
  * It tracks the following resources: Memory, EventLoop, API and CPU.
17
- * The class is used by the {@link AutoscaledPool} class.
17
+ * The class is used by the {@apilink AutoscaledPool} class.
18
18
  *
19
19
  * When running on the Apify platform, the CPU and memory statistics are provided by the platform,
20
20
  * as collected from the running Docker container. When running locally, `Snapshotter`
@@ -29,7 +29,7 @@ export interface SystemInfo {
29
29
  }
30
30
  export interface SystemStatusOptions {
31
31
  /**
32
- * Defines max age of snapshots used in the {@link SystemStatus.getCurrentStatus} measurement.
32
+ * Defines max age of snapshots used in the {@apilink SystemStatus.getCurrentStatus} measurement.
33
33
  * @default 5
34
34
  */
35
35
  currentHistorySecs?: number;
@@ -82,25 +82,25 @@ export interface FinalStatistics {
82
82
  crawlerRuntimeMillis: number;
83
83
  }
84
84
  /**
85
- * Provides a simple interface to reading system status from a {@link Snapshotter} instance.
86
- * It only exposes two functions {@link SystemStatus.getCurrentStatus}
87
- * and {@link SystemStatus.getHistoricalStatus}.
85
+ * Provides a simple interface to reading system status from a {@apilink Snapshotter} instance.
86
+ * It only exposes two functions {@apilink SystemStatus.getCurrentStatus}
87
+ * and {@apilink SystemStatus.getHistoricalStatus}.
88
88
  * The system status is calculated using a weighted average of overloaded
89
89
  * messages in the snapshots, with the weights being the time intervals
90
90
  * between the snapshots. Each resource is calculated separately
91
91
  * and the system is overloaded whenever at least one resource is overloaded.
92
- * The class is used by the {@link AutoscaledPool} class.
92
+ * The class is used by the {@apilink AutoscaledPool} class.
93
93
  *
94
- * {@link SystemStatus.getCurrentStatus}
94
+ * {@apilink SystemStatus.getCurrentStatus}
95
95
  * returns a boolean that represents the current status of the system.
96
96
  * The length of the current timeframe in seconds is configurable
97
97
  * by the `currentHistorySecs` option and represents the max age
98
98
  * of snapshots to be considered for the calculation.
99
99
  *
100
- * {@link SystemStatus.getHistoricalStatus}
100
+ * {@apilink SystemStatus.getHistoricalStatus}
101
101
  * returns a boolean that represents the long-term status
102
102
  * of the system. It considers the full snapshot history available
103
- * in the {@link Snapshotter} instance.
103
+ * in the {@apilink Snapshotter} instance.
104
104
  * @category Scaling
105
105
  */
106
106
  export declare class SystemStatus {
@@ -112,7 +112,7 @@ export declare class SystemStatus {
112
112
  private readonly snapshotter;
113
113
  constructor(options?: SystemStatusOptions);
114
114
  /**
115
- * Returns an {@link SystemInfo} object with the following structure:
115
+ * Returns an {@apilink SystemInfo} object with the following structure:
116
116
  *
117
117
  * ```javascript
118
118
  * {
@@ -129,7 +129,7 @@ export declare class SystemStatus {
129
129
  */
130
130
  getCurrentStatus(): SystemInfo;
131
131
  /**
132
- * Returns an {@link SystemInfo} object with the following structure:
132
+ * Returns an {@apilink SystemInfo} object with the following structure:
133
133
  *
134
134
  * ```javascript
135
135
  * {
@@ -141,8 +141,8 @@ export declare class SystemStatus {
141
141
  * ```
142
142
  *
143
143
  * Where the `isSystemIdle` property is set to `false` if the system
144
- * has been overloaded in the full history of the {@link Snapshotter}
145
- * (which is configurable in the {@link Snapshotter}) and `true` otherwise.
144
+ * has been overloaded in the full history of the {@apilink Snapshotter}
145
+ * (which is configurable in the {@apilink Snapshotter}) and `true` otherwise.
146
146
  */
147
147
  getHistoricalStatus(): SystemInfo;
148
148
  /**
@@ -6,25 +6,25 @@ const ow_1 = tslib_1.__importDefault(require("ow"));
6
6
  const utils_1 = require("@crawlee/utils");
7
7
  const snapshotter_1 = require("./snapshotter");
8
8
  /**
9
- * Provides a simple interface to reading system status from a {@link Snapshotter} instance.
10
- * It only exposes two functions {@link SystemStatus.getCurrentStatus}
11
- * and {@link SystemStatus.getHistoricalStatus}.
9
+ * Provides a simple interface to reading system status from a {@apilink Snapshotter} instance.
10
+ * It only exposes two functions {@apilink SystemStatus.getCurrentStatus}
11
+ * and {@apilink SystemStatus.getHistoricalStatus}.
12
12
  * The system status is calculated using a weighted average of overloaded
13
13
  * messages in the snapshots, with the weights being the time intervals
14
14
  * between the snapshots. Each resource is calculated separately
15
15
  * and the system is overloaded whenever at least one resource is overloaded.
16
- * The class is used by the {@link AutoscaledPool} class.
16
+ * The class is used by the {@apilink AutoscaledPool} class.
17
17
  *
18
- * {@link SystemStatus.getCurrentStatus}
18
+ * {@apilink SystemStatus.getCurrentStatus}
19
19
  * returns a boolean that represents the current status of the system.
20
20
  * The length of the current timeframe in seconds is configurable
21
21
  * by the `currentHistorySecs` option and represents the max age
22
22
  * of snapshots to be considered for the calculation.
23
23
  *
24
- * {@link SystemStatus.getHistoricalStatus}
24
+ * {@apilink SystemStatus.getHistoricalStatus}
25
25
  * returns a boolean that represents the long-term status
26
26
  * of the system. It considers the full snapshot history available
27
- * in the {@link Snapshotter} instance.
27
+ * in the {@apilink Snapshotter} instance.
28
28
  * @category Scaling
29
29
  */
30
30
  class SystemStatus {
@@ -83,7 +83,7 @@ class SystemStatus {
83
83
  this.snapshotter = snapshotter || new snapshotter_1.Snapshotter({ config });
84
84
  }
85
85
  /**
86
- * Returns an {@link SystemInfo} object with the following structure:
86
+ * Returns an {@apilink SystemInfo} object with the following structure:
87
87
  *
88
88
  * ```javascript
89
89
  * {
@@ -102,7 +102,7 @@ class SystemStatus {
102
102
  return this._isSystemIdle(this.currentHistorySecs);
103
103
  }
104
104
  /**
105
- * Returns an {@link SystemInfo} object with the following structure:
105
+ * Returns an {@apilink SystemInfo} object with the following structure:
106
106
  *
107
107
  * ```javascript
108
108
  * {
@@ -114,8 +114,8 @@ class SystemStatus {
114
114
  * ```
115
115
  *
116
116
  * Where the `isSystemIdle` property is set to `false` if the system
117
- * has been overloaded in the full history of the {@link Snapshotter}
118
- * (which is configurable in the {@link Snapshotter}) and `true` otherwise.
117
+ * has been overloaded in the full history of the {@apilink Snapshotter}
118
+ * (which is configurable in the {@apilink Snapshotter}) and `true` otherwise.
119
119
  */
120
120
  getHistoricalStatus() {
121
121
  return this._isSystemIdle();
@@ -8,17 +8,17 @@ import type { EventManager } from './events';
8
8
  export interface ConfigurationOptions {
9
9
  /**
10
10
  * Defines storage client to be used.
11
- * @default {@link MemoryStorage}
11
+ * @default {@apilink MemoryStorage}
12
12
  */
13
13
  storageClient?: StorageClient;
14
14
  /**
15
15
  * Defines the Event Manager to be used.
16
- * @default {@link EventManager}
16
+ * @default {@apilink EventManager}
17
17
  */
18
18
  eventManager?: EventManager;
19
19
  /**
20
20
  * Could be used to adjust the storage client behavior
21
- * e.g. {@link MemoryStorageOptions} could be used to adjust the {@link MemoryStorage} behavior.
21
+ * e.g. {@apilink MemoryStorageOptions} could be used to adjust the {@apilink MemoryStorage} behavior.
22
22
  */
23
23
  storageClientOptions?: Dictionary;
24
24
  /**
@@ -56,7 +56,7 @@ export interface ConfigurationOptions {
56
56
  */
57
57
  maxUsedCpuRatio?: number;
58
58
  /**
59
- * Sets the ratio, defining the amount of system memory that could be used by the {@link AutoscaledPool}.
59
+ * Sets the ratio, defining the amount of system memory that could be used by the {@apilink AutoscaledPool}.
60
60
  * When the memory usage is more than the provided ratio, the memory is considered overloaded.
61
61
  *
62
62
  * Alternative to `CRAWLEE_AVAILABLE_MEMORY_RATIO` environment variable.
@@ -64,7 +64,7 @@ export interface ConfigurationOptions {
64
64
  */
65
65
  availableMemoryRatio?: number;
66
66
  /**
67
- * Sets the amount of system memory in megabytes to be used by the {@link AutoscaledPool}.
67
+ * Sets the amount of system memory in megabytes to be used by the {@apilink AutoscaledPool}.
68
68
  * By default, the maximum memory is set to one quarter of total system memory.
69
69
  *
70
70
  * Alternative to `CRAWLEE_MEMORY_MBYTES` environment variable.
@@ -84,7 +84,7 @@ export interface ConfigurationOptions {
84
84
  systemInfoIntervalMillis?: number;
85
85
  /**
86
86
  * Defines the default input key, i.e. the key that is used to get the crawler input value
87
- * from the default {@link KeyValueStore} associated with the current crawler run.
87
+ * from the default {@apilink KeyValueStore} associated with the current crawler run.
88
88
  *
89
89
  * Alternative to `CRAWLEE_INPUT_KEY` environment variable.
90
90
  * @default 'INPUT'
@@ -224,8 +224,8 @@ export declare class Configuration {
224
224
  */
225
225
  set(key: keyof ConfigurationOptions, value?: any): void;
226
226
  /**
227
- * Returns cached instance of {@link StorageClient} using options as defined in the environment variables or in
228
- * this {@link Configuration} instance. Only first call of this method will create the client, following calls will
227
+ * Returns cached instance of {@apilink StorageClient} using options as defined in the environment variables or in
228
+ * this {@apilink Configuration} instance. Only first call of this method will create the client, following calls will
229
229
  * return the same client instance.
230
230
  *
231
231
  * Caching works based on the `storageClientOptions`, so calling this method with different options will return
@@ -246,11 +246,11 @@ export declare class Configuration {
246
246
  */
247
247
  static getGlobalConfig(): Configuration;
248
248
  /**
249
- * Gets default {@link StorageClient} instance.
249
+ * Gets default {@apilink StorageClient} instance.
250
250
  */
251
251
  static getStorageClient(): StorageClient;
252
252
  /**
253
- * Gets default {@link EventManager} instance.
253
+ * Gets default {@apilink EventManager} instance.
254
254
  */
255
255
  static getEventManager(): EventManager;
256
256
  /**
package/configuration.js CHANGED
@@ -141,8 +141,8 @@ class Configuration {
141
141
  this.options.set(key, value);
142
142
  }
143
143
  /**
144
- * Returns cached instance of {@link StorageClient} using options as defined in the environment variables or in
145
- * this {@link Configuration} instance. Only first call of this method will create the client, following calls will
144
+ * Returns cached instance of {@apilink StorageClient} using options as defined in the environment variables or in
145
+ * this {@apilink Configuration} instance. Only first call of this method will create the client, following calls will
146
146
  * return the same client instance.
147
147
  *
148
148
  * Caching works based on the `storageClientOptions`, so calling this method with different options will return
@@ -197,13 +197,13 @@ class Configuration {
197
197
  return Configuration.globalConfig;
198
198
  }
199
199
  /**
200
- * Gets default {@link StorageClient} instance.
200
+ * Gets default {@apilink StorageClient} instance.
201
201
  */
202
202
  static getStorageClient() {
203
203
  return this.getGlobalConfig().getStorageClient();
204
204
  }
205
205
  /**
206
- * Gets default {@link EventManager} instance.
206
+ * Gets default {@apilink EventManager} instance.
207
207
  */
208
208
  static getEventManager() {
209
209
  return this.getGlobalConfig().getEventManager();
@@ -6,13 +6,13 @@ import type { Session } from '../session_pool/session';
6
6
  export interface CrawlingContext<UserData extends Dictionary = Dictionary> extends Record<PropertyKey, unknown> {
7
7
  id: string;
8
8
  /**
9
- * The original {@link Request} object.
9
+ * The original {@apilink Request} object.
10
10
  */
11
11
  request: Request<UserData>;
12
12
  session?: Session;
13
13
  /**
14
14
  * An object with information about currently used proxy by the crawler
15
- * and configured by the {@link ProxyConfiguration} class.
15
+ * and configured by the {@apilink ProxyConfiguration} class.
16
16
  */
17
17
  proxyInfo?: ProxyInfo;
18
18
  log: Log;
@@ -28,7 +28,7 @@ export declare class Statistics {
28
28
  */
29
29
  readonly id: number;
30
30
  /**
31
- * Current statistic state used for doing calculations on {@link Statistics.calculate} calls
31
+ * Current statistic state used for doing calculations on {@apilink Statistics.calculate} calls
32
32
  */
33
33
  state: StatisticState;
34
34
  /**
@@ -67,7 +67,7 @@ class Statistics {
67
67
  value: Statistics.id++
68
68
  }); // assign an id while incrementing so it can be saved/restored from KV
69
69
  /**
70
- * Current statistic state used for doing calculations on {@link Statistics.calculate} calls
70
+ * Current statistic state used for doing calculations on {@apilink Statistics.calculate} calls
71
71
  */
72
72
  Object.defineProperty(this, "state", {
73
73
  enumerable: true,
@@ -10,9 +10,9 @@ export interface EnqueueLinksOptions {
10
10
  requestQueue: RequestQueue;
11
11
  /** A CSS selector matching links to be enqueued. */
12
12
  selector?: string;
13
- /** Sets {@link Request.userData} for newly enqueued requests. */
13
+ /** Sets {@apilink Request.userData} for newly enqueued requests. */
14
14
  userData?: Dictionary;
15
- /** Sets {@link Request.label} for newly enqueued requests. */
15
+ /** Sets {@apilink Request.label} for newly enqueued requests. */
16
16
  label?: string;
17
17
  /**
18
18
  * A base URL that will be used to resolve relative URLs when using Cheerio. Ignored when using Puppeteer,
@@ -24,7 +24,7 @@ export interface EnqueueLinksOptions {
24
24
  * containing glob pattern strings matching the URLs to be enqueued.
25
25
  *
26
26
  * The plain objects must include at least the `glob` property, which holds the glob pattern string.
27
- * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
27
+ * All remaining keys will be used as request options for the corresponding enqueued {@apilink Request} objects.
28
28
  *
29
29
  * The matching is always case-insensitive.
30
30
  * If you need case-sensitive matching, use `regexps` property directly.
@@ -38,7 +38,7 @@ export interface EnqueueLinksOptions {
38
38
  * containing regular expressions matching the URLs to be enqueued.
39
39
  *
40
40
  * The plain objects must include at least the `regexp` property, which holds the regular expression.
41
- * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
41
+ * All remaining keys will be used as request options for the corresponding enqueued {@apilink Request} objects.
42
42
  *
43
43
  * If `regexps` is an empty array or `undefined`, and `globs` are also not defined, then the function
44
44
  * enqueues the links with the same subdomain.
@@ -48,11 +48,11 @@ export interface EnqueueLinksOptions {
48
48
  * *NOTE:* In future versions of SDK the options will be removed.
49
49
  * Please use `globs` or `regexps` instead.
50
50
  *
51
- * An array of {@link PseudoUrl} strings or plain objects
52
- * containing {@link PseudoUrl} strings matching the URLs to be enqueued.
51
+ * An array of {@apilink PseudoUrl} strings or plain objects
52
+ * containing {@apilink PseudoUrl} strings matching the URLs to be enqueued.
53
53
  *
54
54
  * The plain objects must include at least the `purl` property, which holds the pseudo-URL string.
55
- * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
55
+ * All remaining keys will be used as request options for the corresponding enqueued {@apilink Request} objects.
56
56
  *
57
57
  * With a pseudo-URL string, the matching is always case-insensitive.
58
58
  * If you need case-sensitive matching, use `regexps` property directly.
@@ -64,7 +64,7 @@ export interface EnqueueLinksOptions {
64
64
  */
65
65
  pseudoUrls?: PseudoUrlInput[];
66
66
  /**
67
- * Just before a new {@link Request} is constructed and enqueued to the {@link RequestQueue}, this function can be used
67
+ * Just before a new {@apilink Request} is constructed and enqueued to the {@apilink RequestQueue}, this function can be used
68
68
  * to remove it or modify its contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
69
69
  * when you need to enqueue multiple `Requests` to the queue that share the same URL, but differ in methods or payloads,
70
70
  * or to dynamically update or create `userData`.
@@ -113,11 +113,11 @@ export declare enum EnqueueStrategy {
113
113
  SameDomain = "same-domain"
114
114
  }
115
115
  /**
116
- * This function enqueues the urls provided to the {@link RequestQueue} provided. If you want to automatically find and enqueue links,
116
+ * This function enqueues the urls provided to the {@apilink RequestQueue} provided. If you want to automatically find and enqueue links,
117
117
  * you should use the context-aware `enqueueLinks` function provided on the crawler contexts.
118
118
  *
119
119
  * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
120
- * and override settings of the enqueued {@link Request} objects.
120
+ * and override settings of the enqueued {@apilink Request} objects.
121
121
  *
122
122
  * **Example usage**
123
123
  *
@@ -134,14 +134,15 @@ export declare enum EnqueueStrategy {
134
134
  * ```
135
135
  *
136
136
  * @param options All `enqueueLinks()` parameters are passed via an options object.
137
- * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
137
+ * @returns Promise that resolves to {@apilink BatchAddRequestsResult} object.
138
138
  */
139
139
  export declare function enqueueLinks(options: EnqueueLinksOptions): Promise<BatchAddRequestsResult>;
140
140
  /**
141
141
  * @internal
142
- * This method helps resolve the baseUrl that will be used for filtering in {@link enqueueLinks}.
142
+ * This method helps resolve the baseUrl that will be used for filtering in {@apilink enqueueLinks}.
143
143
  * - If a user provides a base url, we always return it
144
- * - If a user specifies {@link EnqueueStrategy.All} strategy, they do not care if the newly found urls are on the original request domain, or a redirected one
144
+ * - If a user specifies {@apilink EnqueueStrategy.All} strategy, they do not care if the newly found urls are on the original
145
+ * request domain, or a redirected one
145
146
  * - In all other cases, we return the domain of the original request as that's the one we need to use for filtering
146
147
  */
147
148
  export declare function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalRequestUrl, originalRequestUrl, userProvidedBaseUrl, }: ResolveBaseUrl): string | undefined;
@@ -1 +1 @@
1
- {"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,gBAAgB,EAAoB,MAAM,UAAU,CAAC;AAQ3G,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAG9D,MAAM,WAAW,mBAAmB;IAChC,sHAAsH;IACtH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,EAAE,MAAM,EAAE,CAAC;IAEf,0DAA0D;IAC1D,YAAY,EAAE,YAAY,CAAC;IAE3B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,iEAAiE;IACjE,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB,8DAA8D;IAC9D,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IAEpB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IAExB;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU,CAAC,EAAE,cAAc,EAAE,CAAC;IAE9B;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;OAGG;IACH,QAAQ,CAAC,EAAE,eAAe,GAAG,KAAK,GAAG,aAAa,GAAG,eAAe,CAAC;CACxE;AAED,oBAAY,eAAe;IACvB;;OAEG;IACH,GAAG,QAAQ;IAEX;;;;OAIG;IACH,YAAY,kBAAkB;IAE9B;;;;OAIG;IACH,UAAU,gBAAgB;CAC7B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAuGhG;AAED;;;;;;GAMG;AACH,wBAAgB,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACtB,EAAE,cAAc,sBA+BhB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,eAAe,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;IAClD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B"}
1
+ {"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,gBAAgB,EAAoB,MAAM,UAAU,CAAC;AAQ3G,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAG9D,MAAM,WAAW,mBAAmB;IAChC,sHAAsH;IACtH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,EAAE,MAAM,EAAE,CAAC;IAEf,0DAA0D;IAC1D,YAAY,EAAE,YAAY,CAAC;IAE3B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,oEAAoE;IACpE,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB,iEAAiE;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IAEpB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IAExB;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU,CAAC,EAAE,cAAc,EAAE,CAAC;IAE9B;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;OAGG;IACH,QAAQ,CAAC,EAAE,eAAe,GAAG,KAAK,GAAG,aAAa,GAAG,eAAe,CAAC;CACxE;AAED,oBAAY,eAAe;IACvB;;OAEG;IACH,GAAG,QAAQ;IAEX;;;;OAIG;IACH,YAAY,kBAAkB;IAE9B;;;;OAIG;IACH,UAAU,gBAAgB;CAC7B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAuGhG;AAED;;;;;;;GAOG;AACH,wBAAgB,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACtB,EAAE,cAAc,sBA+BhB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,eAAe,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;IAClD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B"}
@@ -26,11 +26,11 @@ var EnqueueStrategy;
26
26
  EnqueueStrategy["SameDomain"] = "same-domain";
27
27
  })(EnqueueStrategy = exports.EnqueueStrategy || (exports.EnqueueStrategy = {}));
28
28
  /**
29
- * This function enqueues the urls provided to the {@link RequestQueue} provided. If you want to automatically find and enqueue links,
29
+ * This function enqueues the urls provided to the {@apilink RequestQueue} provided. If you want to automatically find and enqueue links,
30
30
  * you should use the context-aware `enqueueLinks` function provided on the crawler contexts.
31
31
  *
32
32
  * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
33
- * and override settings of the enqueued {@link Request} objects.
33
+ * and override settings of the enqueued {@apilink Request} objects.
34
34
  *
35
35
  * **Example usage**
36
36
  *
@@ -47,7 +47,7 @@ var EnqueueStrategy;
47
47
  * ```
48
48
  *
49
49
  * @param options All `enqueueLinks()` parameters are passed via an options object.
50
- * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
50
+ * @returns Promise that resolves to {@apilink BatchAddRequestsResult} object.
51
51
  */
52
52
  async function enqueueLinks(options) {
53
53
  (0, ow_1.default)(options, ow_1.default.object.exactShape({
@@ -124,9 +124,10 @@ async function enqueueLinks(options) {
124
124
  exports.enqueueLinks = enqueueLinks;
125
125
  /**
126
126
  * @internal
127
- * This method helps resolve the baseUrl that will be used for filtering in {@link enqueueLinks}.
127
+ * This method helps resolve the baseUrl that will be used for filtering in {@apilink enqueueLinks}.
128
128
  * - If a user provides a base url, we always return it
129
- * - If a user specifies {@link EnqueueStrategy.All} strategy, they do not care if the newly found urls are on the original request domain, or a redirected one
129
+ * - If a user specifies {@apilink EnqueueStrategy.All} strategy, they do not care if the newly found urls are on the original
130
+ * request domain, or a redirected one
130
131
  * - In all other cases, we return the domain of the original request as that's the one we need to use for filtering
131
132
  */
132
133
  function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalRequestUrl, originalRequestUrl, userProvidedBaseUrl, }) {
@@ -1 +1 @@
1
- {"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":";;;;AAAA,iCAAkC;AAClC,oDAAoB;AACpB,6DAA6B;AAG7B,qCAMkB;AA6GlB,IAAY,eAmBX;AAnBD,WAAY,eAAe;IACvB;;OAEG;IACH,8BAAW,CAAA;IAEX;;;;OAIG;IACH,iDAA8B,CAAA;IAE9B;;;;OAIG;IACH,6CAA0B,CAAA;AAC9B,CAAC,EAnBW,eAAe,GAAf,uBAAe,KAAf,uBAAe,QAmB1B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACI,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC3D,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,IAAI,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QAChC,YAAY,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC;QACjE,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACvC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAClC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACpC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAC9B,CAAC;QACF,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC9C,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC;YAC/B,eAAe,CAAC,GAAG;YACnB,eAAe,CAAC,YAAY;YAC5B,eAAe,CAAC,UAAU;SAC7B,CAAC;KACL,CAAC,CAAC,CAAC;IAEJ,MAAM,EACF,YAAY,EACZ,KAAK,EACL,IAAI,EACJ,UAAU,EACV,KAAK,EACL,OAAO,EACP,wBAAwB,GAC3B,GAAG,OAAO,CAAC;IAEZ,MAAM,iBAAiB,GAAuB,EAAE,CAAC;IAEjD,IAAI,UAAU,EAAE,MAAM,EAAE;QACpB,aAAG,CAAC,UAAU,CAAC,qEAAqE,CAAC,CAAC;QACtF,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,6CAAoC,EAAC,UAAU,CAAC,CAAC,CAAC;KAC/E;IAED,IAAI,KAAK,EAAE,MAAM,EAAE;QACf,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,sCAA6B,EAAC,KAAK,CAAC,CAAC,CAAC;KACnE;IAED,IAAI,OAAO,EAAE,MAAM,EAAE;QACjB,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,0CAAiC,EAAC,OAAO,CAAC,CAAC,CAAC;KACzE;IAED,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE;QAC3B,OAAO,CAAC,QAAQ,KAAhB,OAAO,CAAC,QAAQ,GAAK,eAAe,CAAC,YAAY,EAAC;KACrD;IAED,IAAI,OAAO,CAAC,OAAO,EAAE;QACjB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAErC,QAAQ,OAAO,CAAC,QAAQ,EAAE;YACtB,KAAK,eAAe,CAAC,YAAY;gBAC7B,sFAAsF;gBACtF,uFAAuF;gBACvF,yCAAyC;gBACzC,iBAAiB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;gBACrD,MAAM;YACV,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC;gBAC7B,4CAA4C;gBAC5C,MAAM,eAAe,GAAG,IAAA,iBAAS,EAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;gBAExE,IAAI,eAAe,EAAE;oBACjB,gHAAgH;oBAChH,GAAG,CAAC,QAAQ,GAAG,eAAe,CAAC;oBAC/B,iBAAiB,CAAC,IAAI,CAClB,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,KAAK,eAAe,EAAE,CAAC,KAAK,EAAE,EAC7E,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAC/B,CAAC;iBACL;qBAAM;oBACH,6FAA6F;oBAC7F,4BAA4B;oBAC5B,iBAAiB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;iBACxD;gBAED,MAAM;aACT;YACD,KAAK,eAAe,CAAC,GAAG,CAAC;YACzB;gBACI,MAAM;SACb;KACJ;IAED,IAAI,cAAc,GAAG,IAAA,6BAAoB,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAEzD,IAAI,wBAAwB,EAAE;QAC1B,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,wBAAwB,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAqB,CAAC;KAC9H;IAED,IAAI,QAAQ,GAAG,IAAA,uBAAc,EAAC,cAAc,EAAE,iBAAiB,CAAC,CAAC;IACjE,IAAI,KAAK;QAAE,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAE/C,OAAO,YAAY,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;AAC9C,CAAC;AAvGD,oCAuGC;AAED;;;;;;GAMG;AACH,SAAgB,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACN;IACb,wCAAwC;IACxC,IAAI,mBAAmB,EAAE;QACrB,OAAO,mBAAmB,CAAC;KAC9B;IAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAC7D,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,eAAe,IAAI,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAE7E,6DAA6D;IAC7D,IAAI,eAAe,KAAK,eAAe,CAAC,GAAG,EAAE;QACzC,OAAO,cAAc,CAAC;KACzB;IAED,0HAA0H;IAC1H,+IAA+I;IAC/I,4DAA4D;IAC5D,IAAI,eAAe,KAAK,eAAe,CAAC,UAAU,EAAE;QAChD,MAAM,gBAAgB,GAAG,IAAA,iBAAS,EAAC,iBAAiB,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAC/E,MAAM,aAAa,GAAG,IAAA,iBAAS,EAAC,cAAc,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAEzE,IAAI,gBAAgB,KAAK,aAAa,EAAE;YACpC,OAAO,cAAc,CAAC;SACzB;QAED,OAAO,SAAS,CAAC;KACpB;IAED,2JAA2J;IAC3J,mCAAmC;IACnC,OAAO,iBAAiB,CAAC;AAC7B,CAAC;AApCD,wFAoCC"}
1
+ {"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":";;;;AAAA,iCAAkC;AAClC,oDAAoB;AACpB,6DAA6B;AAG7B,qCAMkB;AA6GlB,IAAY,eAmBX;AAnBD,WAAY,eAAe;IACvB;;OAEG;IACH,8BAAW,CAAA;IAEX;;;;OAIG;IACH,iDAA8B,CAAA;IAE9B;;;;OAIG;IACH,6CAA0B,CAAA;AAC9B,CAAC,EAnBW,eAAe,GAAf,uBAAe,KAAf,uBAAe,QAmB1B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACI,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC3D,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,IAAI,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QAChC,YAAY,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC;QACjE,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACvC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAClC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACpC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAC9B,CAAC;QACF,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC9C,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC;YAC/B,eAAe,CAAC,GAAG;YACnB,eAAe,CAAC,YAAY;YAC5B,eAAe,CAAC,UAAU;SAC7B,CAAC;KACL,CAAC,CAAC,CAAC;IAEJ,MAAM,EACF,YAAY,EACZ,KAAK,EACL,IAAI,EACJ,UAAU,EACV,KAAK,EACL,OAAO,EACP,wBAAwB,GAC3B,GAAG,OAAO,CAAC;IAEZ,MAAM,iBAAiB,GAAuB,EAAE,CAAC;IAEjD,IAAI,UAAU,EAAE,MAAM,EAAE;QACpB,aAAG,CAAC,UAAU,CAAC,qEAAqE,CAAC,CAAC;QACtF,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,6CAAoC,EAAC,UAAU,CAAC,CAAC,CAAC;KAC/E;IAED,IAAI,KAAK,EAAE,MAAM,EAAE;QACf,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,sCAA6B,EAAC,KAAK,CAAC,CAAC,CAAC;KACnE;IAED,IAAI,OAAO,EAAE,MAAM,EAAE;QACjB,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,0CAAiC,EAAC,OAAO,CAAC,CAAC,CAAC;KACzE;IAED,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE;QAC3B,OAAO,CAAC,QAAQ,KAAhB,OAAO,CAAC,QAAQ,GAAK,eAAe,CAAC,YAAY,EAAC;KACrD;IAED,IAAI,OAAO,CAAC,OAAO,EAAE;QACjB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAErC,QAAQ,OAAO,CAAC,QAAQ,EAAE;YACtB,KAAK,eAAe,CAAC,YAAY;gBAC7B,sFAAsF;gBACtF,uFAAuF;gBACvF,yCAAyC;gBACzC,iBAAiB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;gBACrD,MAAM;YACV,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC;gBAC7B,4CAA4C;gBAC5C,MAAM,eAAe,GAAG,IAAA,iBAAS,EAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;gBAExE,IAAI,eAAe,EAAE;oBACjB,gHAAgH;oBAChH,GAAG,CAAC,QAAQ,GAAG,eAAe,CAAC;oBAC/B,iBAAiB,CAAC,IAAI,CAClB,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,KAAK,eAAe,EAAE,CAAC,KAAK,EAAE,EAC7E,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAC/B,CAAC;iBACL;qBAAM;oBACH,6FAA6F;oBAC7F,4BAA4B;oBAC5B,iBAAiB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;iBACxD;gBAED,MAAM;aACT;YACD,KAAK,eAAe,CAAC,GAAG,CAAC;YACzB;gBACI,MAAM;SACb;KACJ;IAED,IAAI,cAAc,GAAG,IAAA,6BAAoB,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAEzD,IAAI,wBAAwB,EAAE;QAC1B,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,wBAAwB,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAqB,CAAC;KAC9H;IAED,IAAI,QAAQ,GAAG,IAAA,uBAAc,EAAC,cAAc,EAAE,iBAAiB,CAAC,CAAC;IACjE,IAAI,KAAK;QAAE,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAE/C,OAAO,YAAY,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;AAC9C,CAAC;AAvGD,oCAuGC;AAED;;;;;;;GAOG;AACH,SAAgB,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACN;IACb,wCAAwC;IACxC,IAAI,mBAAmB,EAAE;QACrB,OAAO,mBAAmB,CAAC;KAC9B;IAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAC7D,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,eAAe,IAAI,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAE7E,6DAA6D;IAC7D,IAAI,eAAe,KAAK,eAAe,CAAC,GAAG,EAAE;QACzC,OAAO,cAAc,CAAC;KACzB;IAED,0HAA0H;IAC1H,+IAA+I;IAC/I,4DAA4D;IAC5D,IAAI,eAAe,KAAK,eAAe,CAAC,UAAU,EAAE;QAChD,MAAM,gBAAgB,GAAG,IAAA,iBAAS,EAAC,iBAAiB,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAC/E,MAAM,aAAa,GAAG,IAAA,iBAAS,EAAC,cAAc,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAEzE,IAAI,gBAAgB,KAAK,aAAa,EAAE;YACpC,OAAO,cAAc,CAAC;SACzB;QAED,OAAO,SAAS,CAAC;KACpB;IAED,2JAA2J;IAC3J,mCAAmC;IACnC,OAAO,iBAAiB,CAAC;AAC7B,CAAC;AApCD,wFAoCC"}
@@ -52,8 +52,8 @@ export declare function createRequests(requestOptions: (string | RequestOptions)
52
52
  */
53
53
  export declare function createRequestOptions(sources: (string | Record<string, unknown>)[], options?: Pick<EnqueueLinksOptions, 'label' | 'userData'>): RequestOptions[];
54
54
  /**
55
- * Takes an Apify {@link RequestOptions} object and changes its attributes in a desired way. This user-function is used
56
- * {@link enqueueLinks} to modify requests before enqueuing them.
55
+ * Takes an Apify {@apilink RequestOptions} object and changes its attributes in a desired way. This user-function is used
56
+ * {@apilink enqueueLinks} to modify requests before enqueuing them.
57
57
  */
58
58
  export interface RequestTransform {
59
59
  /**
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/core",
3
- "version": "3.0.3-beta.10",
3
+ "version": "3.0.3-beta.13",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=16.0.0"
@@ -58,9 +58,9 @@
58
58
  "@apify/pseudo_url": "^2.0.0",
59
59
  "@apify/timeout": "^0.3.0",
60
60
  "@apify/utilities": "^2.0.0",
61
- "@crawlee/memory-storage": "^3.0.3-beta.10",
62
- "@crawlee/types": "^3.0.3-beta.10",
63
- "@crawlee/utils": "^3.0.3-beta.10",
61
+ "@crawlee/memory-storage": "^3.0.3-beta.13",
62
+ "@crawlee/types": "^3.0.3-beta.13",
63
+ "@crawlee/utils": "^3.0.3-beta.13",
64
64
  "@types/tough-cookie": "^4.0.2",
65
65
  "@vladfrangu/async_event_emitter": "^2.0.0",
66
66
  "fs-extra": "^10.1.0",