@crawlee/core 3.0.3-beta.10 → 3.0.3-beta.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/autoscaled_pool.d.ts +16 -16
- package/autoscaling/autoscaled_pool.js +13 -13
- package/autoscaling/snapshotter.d.ts +1 -1
- package/autoscaling/snapshotter.js +1 -1
- package/autoscaling/system_status.d.ts +12 -12
- package/autoscaling/system_status.js +11 -11
- package/configuration.d.ts +10 -10
- package/configuration.js +4 -4
- package/crawlers/crawler_commons.d.ts +2 -2
- package/crawlers/statistics.d.ts +1 -1
- package/crawlers/statistics.js +1 -1
- package/enqueue_links/enqueue_links.d.ts +14 -13
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +6 -5
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +2 -2
- package/package.json +4 -4
- package/proxy_configuration.d.ts +11 -11
- package/proxy_configuration.js +8 -8
- package/request.d.ts +3 -3
- package/request.js +2 -2
- package/session_pool/session.d.ts +1 -1
- package/session_pool/session_pool.d.ts +12 -12
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +10 -10
- package/storages/dataset.d.ts +14 -14
- package/storages/dataset.js +8 -8
- package/storages/key_value_store.d.ts +24 -24
- package/storages/key_value_store.js +20 -20
- package/storages/request_list.d.ts +33 -33
- package/storages/request_list.js +23 -23
- package/storages/request_queue.d.ts +25 -25
- package/storages/request_queue.js +23 -23
- package/storages/utils.d.ts +1 -1
- package/storages/utils.js +1 -1
- package/tsconfig.build.tsbuildinfo +1 -1
|
@@ -20,7 +20,7 @@ export interface AutoscaledPoolOptions {
|
|
|
20
20
|
* If it resolves to `true` then the pool's run finishes. Being called only
|
|
21
21
|
* when there are no tasks being processed means that as long as `isTaskReadyFunction()`
|
|
22
22
|
* keeps resolving to `true`, `isFinishedFunction()` will never be called.
|
|
23
|
-
* To abort a run, use the {@
|
|
23
|
+
* To abort a run, use the {@apilink AutoscaledPool.abort} method.
|
|
24
24
|
*/
|
|
25
25
|
isFinishedFunction?: () => Promise<boolean>;
|
|
26
26
|
/**
|
|
@@ -84,12 +84,12 @@ export interface AutoscaledPoolOptions {
|
|
|
84
84
|
*/
|
|
85
85
|
taskTimeoutSecs?: number;
|
|
86
86
|
/**
|
|
87
|
-
* Options to be passed down to the {@
|
|
87
|
+
* Options to be passed down to the {@apilink Snapshotter} constructor. This is useful for fine-tuning
|
|
88
88
|
* the snapshot intervals and history.
|
|
89
89
|
*/
|
|
90
90
|
snapshotterOptions?: SnapshotterOptions;
|
|
91
91
|
/**
|
|
92
|
-
* Options to be passed down to the {@
|
|
92
|
+
* Options to be passed down to the {@apilink SystemStatus} constructor. This is useful for fine-tuning
|
|
93
93
|
* the system status reports. If a custom snapshotter is set in the options, it will be used
|
|
94
94
|
* by the pool.
|
|
95
95
|
*/
|
|
@@ -106,22 +106,22 @@ export interface AutoscaledPoolOptions {
|
|
|
106
106
|
* The pool only starts new tasks if there is enough free CPU and memory available
|
|
107
107
|
* and the Javascript event loop is not blocked.
|
|
108
108
|
*
|
|
109
|
-
* The information about the CPU and memory usage is obtained by the {@
|
|
109
|
+
* The information about the CPU and memory usage is obtained by the {@apilink Snapshotter} class,
|
|
110
110
|
* which makes regular snapshots of system resources that may be either local
|
|
111
111
|
* or from the Apify cloud infrastructure in case the process is running on the Apify platform.
|
|
112
|
-
* Meaningful data gathered from these snapshots is provided to `AutoscaledPool` by the {@
|
|
112
|
+
* Meaningful data gathered from these snapshots is provided to `AutoscaledPool` by the {@apilink SystemStatus} class.
|
|
113
113
|
*
|
|
114
114
|
* Before running the pool, you need to implement the following three functions:
|
|
115
|
-
* {@
|
|
116
|
-
* {@
|
|
117
|
-
* {@
|
|
115
|
+
* {@apilink AutoscaledPoolOptions.runTaskFunction},
|
|
116
|
+
* {@apilink AutoscaledPoolOptions.isTaskReadyFunction} and
|
|
117
|
+
* {@apilink AutoscaledPoolOptions.isFinishedFunction}.
|
|
118
118
|
*
|
|
119
|
-
* The auto-scaled pool is started by calling the {@
|
|
120
|
-
* The pool periodically queries the {@
|
|
119
|
+
* The auto-scaled pool is started by calling the {@apilink AutoscaledPool.run} function.
|
|
120
|
+
* The pool periodically queries the {@apilink AutoscaledPoolOptions.isTaskReadyFunction} function
|
|
121
121
|
* for more tasks, managing optimal concurrency, until the function resolves to `false`. The pool then queries
|
|
122
|
-
* the {@
|
|
122
|
+
* the {@apilink AutoscaledPoolOptions.isFinishedFunction}. If it resolves to `true`, the run finishes after all running tasks complete.
|
|
123
123
|
* If it resolves to `false`, it assumes there will be more tasks available later and keeps periodically querying for tasks.
|
|
124
|
-
* If any of the tasks throws then the {@
|
|
124
|
+
* If any of the tasks throws then the {@apilink AutoscaledPool.run} function rejects the promise with an error.
|
|
125
125
|
*
|
|
126
126
|
* The pool evaluates whether it should start a new task every time one of the tasks finishes
|
|
127
127
|
* and also in the interval set by the `options.maybeRunIntervalSecs` parameter.
|
|
@@ -221,7 +221,7 @@ export declare class AutoscaledPool {
|
|
|
221
221
|
run(): Promise<void>;
|
|
222
222
|
/**
|
|
223
223
|
* Aborts the run of the auto-scaled pool and destroys it. The promise returned from
|
|
224
|
-
* the {@
|
|
224
|
+
* the {@apilink AutoscaledPool.run} function will immediately resolve, no more new tasks
|
|
225
225
|
* will be spawned and all running tasks will be left in their current state.
|
|
226
226
|
*
|
|
227
227
|
* Due to the nature of the tasks, auto-scaled pool cannot reliably guarantee abortion
|
|
@@ -233,19 +233,19 @@ export declare class AutoscaledPool {
|
|
|
233
233
|
abort(): Promise<void>;
|
|
234
234
|
/**
|
|
235
235
|
* Prevents the auto-scaled pool from starting new tasks, but allows the running ones to finish
|
|
236
|
-
* (unlike abort, which terminates them). Used together with {@
|
|
236
|
+
* (unlike abort, which terminates them). Used together with {@apilink AutoscaledPool.resume}
|
|
237
237
|
*
|
|
238
238
|
* The function's promise will resolve once all running tasks have completed and the pool
|
|
239
239
|
* is effectively idle. If the `timeoutSecs` argument is provided, the promise will reject
|
|
240
240
|
* with a timeout error after the `timeoutSecs` seconds.
|
|
241
241
|
*
|
|
242
|
-
* The promise returned from the {@
|
|
242
|
+
* The promise returned from the {@apilink AutoscaledPool.run} function will not resolve
|
|
243
243
|
* when `.pause()` is invoked (unlike abort, which resolves it).
|
|
244
244
|
*/
|
|
245
245
|
pause(timeoutSecs?: number): Promise<void>;
|
|
246
246
|
/**
|
|
247
247
|
* Resumes the operation of the autoscaled-pool by allowing more tasks to be run.
|
|
248
|
-
* Used together with {@
|
|
248
|
+
* Used together with {@apilink AutoscaledPool.pause}
|
|
249
249
|
*
|
|
250
250
|
* Tasks will automatically start running again in `options.maybeRunIntervalSecs`.
|
|
251
251
|
*/
|
|
@@ -14,22 +14,22 @@ const system_status_1 = require("./system_status");
|
|
|
14
14
|
* The pool only starts new tasks if there is enough free CPU and memory available
|
|
15
15
|
* and the Javascript event loop is not blocked.
|
|
16
16
|
*
|
|
17
|
-
* The information about the CPU and memory usage is obtained by the {@
|
|
17
|
+
* The information about the CPU and memory usage is obtained by the {@apilink Snapshotter} class,
|
|
18
18
|
* which makes regular snapshots of system resources that may be either local
|
|
19
19
|
* or from the Apify cloud infrastructure in case the process is running on the Apify platform.
|
|
20
|
-
* Meaningful data gathered from these snapshots is provided to `AutoscaledPool` by the {@
|
|
20
|
+
* Meaningful data gathered from these snapshots is provided to `AutoscaledPool` by the {@apilink SystemStatus} class.
|
|
21
21
|
*
|
|
22
22
|
* Before running the pool, you need to implement the following three functions:
|
|
23
|
-
* {@
|
|
24
|
-
* {@
|
|
25
|
-
* {@
|
|
23
|
+
* {@apilink AutoscaledPoolOptions.runTaskFunction},
|
|
24
|
+
* {@apilink AutoscaledPoolOptions.isTaskReadyFunction} and
|
|
25
|
+
* {@apilink AutoscaledPoolOptions.isFinishedFunction}.
|
|
26
26
|
*
|
|
27
|
-
* The auto-scaled pool is started by calling the {@
|
|
28
|
-
* The pool periodically queries the {@
|
|
27
|
+
* The auto-scaled pool is started by calling the {@apilink AutoscaledPool.run} function.
|
|
28
|
+
* The pool periodically queries the {@apilink AutoscaledPoolOptions.isTaskReadyFunction} function
|
|
29
29
|
* for more tasks, managing optimal concurrency, until the function resolves to `false`. The pool then queries
|
|
30
|
-
* the {@
|
|
30
|
+
* the {@apilink AutoscaledPoolOptions.isFinishedFunction}. If it resolves to `true`, the run finishes after all running tasks complete.
|
|
31
31
|
* If it resolves to `false`, it assumes there will be more tasks available later and keeps periodically querying for tasks.
|
|
32
|
-
* If any of the tasks throws then the {@
|
|
32
|
+
* If any of the tasks throws then the {@apilink AutoscaledPool.run} function rejects the promise with an error.
|
|
33
33
|
*
|
|
34
34
|
* The pool evaluates whether it should start a new task every time one of the tasks finishes
|
|
35
35
|
* and also in the interval set by the `options.maybeRunIntervalSecs` parameter.
|
|
@@ -378,7 +378,7 @@ class AutoscaledPool {
|
|
|
378
378
|
}
|
|
379
379
|
/**
|
|
380
380
|
* Aborts the run of the auto-scaled pool and destroys it. The promise returned from
|
|
381
|
-
* the {@
|
|
381
|
+
* the {@apilink AutoscaledPool.run} function will immediately resolve, no more new tasks
|
|
382
382
|
* will be spawned and all running tasks will be left in their current state.
|
|
383
383
|
*
|
|
384
384
|
* Due to the nature of the tasks, auto-scaled pool cannot reliably guarantee abortion
|
|
@@ -396,13 +396,13 @@ class AutoscaledPool {
|
|
|
396
396
|
}
|
|
397
397
|
/**
|
|
398
398
|
* Prevents the auto-scaled pool from starting new tasks, but allows the running ones to finish
|
|
399
|
-
* (unlike abort, which terminates them). Used together with {@
|
|
399
|
+
* (unlike abort, which terminates them). Used together with {@apilink AutoscaledPool.resume}
|
|
400
400
|
*
|
|
401
401
|
* The function's promise will resolve once all running tasks have completed and the pool
|
|
402
402
|
* is effectively idle. If the `timeoutSecs` argument is provided, the promise will reject
|
|
403
403
|
* with a timeout error after the `timeoutSecs` seconds.
|
|
404
404
|
*
|
|
405
|
-
* The promise returned from the {@
|
|
405
|
+
* The promise returned from the {@apilink AutoscaledPool.run} function will not resolve
|
|
406
406
|
* when `.pause()` is invoked (unlike abort, which resolves it).
|
|
407
407
|
*/
|
|
408
408
|
async pause(timeoutSecs) {
|
|
@@ -431,7 +431,7 @@ class AutoscaledPool {
|
|
|
431
431
|
}
|
|
432
432
|
/**
|
|
433
433
|
* Resumes the operation of the autoscaled-pool by allowing more tasks to be run.
|
|
434
|
-
* Used together with {@
|
|
434
|
+
* Used together with {@apilink AutoscaledPool.pause}
|
|
435
435
|
*
|
|
436
436
|
* Tasks will automatically start running again in `options.maybeRunIntervalSecs`.
|
|
437
437
|
*/
|
|
@@ -75,7 +75,7 @@ interface ClientSnapshot {
|
|
|
75
75
|
* Creates snapshots of system resources at given intervals and marks the resource
|
|
76
76
|
* as either overloaded or not during the last interval. Keeps a history of the snapshots.
|
|
77
77
|
* It tracks the following resources: Memory, EventLoop, API and CPU.
|
|
78
|
-
* The class is used by the {@
|
|
78
|
+
* The class is used by the {@apilink AutoscaledPool} class.
|
|
79
79
|
*
|
|
80
80
|
* When running on the Apify platform, the CPU and memory statistics are provided by the platform,
|
|
81
81
|
* as collected from the running Docker container. When running locally, `Snapshotter`
|
|
@@ -14,7 +14,7 @@ const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10000;
|
|
|
14
14
|
* Creates snapshots of system resources at given intervals and marks the resource
|
|
15
15
|
* as either overloaded or not during the last interval. Keeps a history of the snapshots.
|
|
16
16
|
* It tracks the following resources: Memory, EventLoop, API and CPU.
|
|
17
|
-
* The class is used by the {@
|
|
17
|
+
* The class is used by the {@apilink AutoscaledPool} class.
|
|
18
18
|
*
|
|
19
19
|
* When running on the Apify platform, the CPU and memory statistics are provided by the platform,
|
|
20
20
|
* as collected from the running Docker container. When running locally, `Snapshotter`
|
|
@@ -29,7 +29,7 @@ export interface SystemInfo {
|
|
|
29
29
|
}
|
|
30
30
|
export interface SystemStatusOptions {
|
|
31
31
|
/**
|
|
32
|
-
* Defines max age of snapshots used in the {@
|
|
32
|
+
* Defines max age of snapshots used in the {@apilink SystemStatus.getCurrentStatus} measurement.
|
|
33
33
|
* @default 5
|
|
34
34
|
*/
|
|
35
35
|
currentHistorySecs?: number;
|
|
@@ -82,25 +82,25 @@ export interface FinalStatistics {
|
|
|
82
82
|
crawlerRuntimeMillis: number;
|
|
83
83
|
}
|
|
84
84
|
/**
|
|
85
|
-
* Provides a simple interface to reading system status from a {@
|
|
86
|
-
* It only exposes two functions {@
|
|
87
|
-
* and {@
|
|
85
|
+
* Provides a simple interface to reading system status from a {@apilink Snapshotter} instance.
|
|
86
|
+
* It only exposes two functions {@apilink SystemStatus.getCurrentStatus}
|
|
87
|
+
* and {@apilink SystemStatus.getHistoricalStatus}.
|
|
88
88
|
* The system status is calculated using a weighted average of overloaded
|
|
89
89
|
* messages in the snapshots, with the weights being the time intervals
|
|
90
90
|
* between the snapshots. Each resource is calculated separately
|
|
91
91
|
* and the system is overloaded whenever at least one resource is overloaded.
|
|
92
|
-
* The class is used by the {@
|
|
92
|
+
* The class is used by the {@apilink AutoscaledPool} class.
|
|
93
93
|
*
|
|
94
|
-
* {@
|
|
94
|
+
* {@apilink SystemStatus.getCurrentStatus}
|
|
95
95
|
* returns a boolean that represents the current status of the system.
|
|
96
96
|
* The length of the current timeframe in seconds is configurable
|
|
97
97
|
* by the `currentHistorySecs` option and represents the max age
|
|
98
98
|
* of snapshots to be considered for the calculation.
|
|
99
99
|
*
|
|
100
|
-
* {@
|
|
100
|
+
* {@apilink SystemStatus.getHistoricalStatus}
|
|
101
101
|
* returns a boolean that represents the long-term status
|
|
102
102
|
* of the system. It considers the full snapshot history available
|
|
103
|
-
* in the {@
|
|
103
|
+
* in the {@apilink Snapshotter} instance.
|
|
104
104
|
* @category Scaling
|
|
105
105
|
*/
|
|
106
106
|
export declare class SystemStatus {
|
|
@@ -112,7 +112,7 @@ export declare class SystemStatus {
|
|
|
112
112
|
private readonly snapshotter;
|
|
113
113
|
constructor(options?: SystemStatusOptions);
|
|
114
114
|
/**
|
|
115
|
-
* Returns an {@
|
|
115
|
+
* Returns an {@apilink SystemInfo} object with the following structure:
|
|
116
116
|
*
|
|
117
117
|
* ```javascript
|
|
118
118
|
* {
|
|
@@ -129,7 +129,7 @@ export declare class SystemStatus {
|
|
|
129
129
|
*/
|
|
130
130
|
getCurrentStatus(): SystemInfo;
|
|
131
131
|
/**
|
|
132
|
-
* Returns an {@
|
|
132
|
+
* Returns an {@apilink SystemInfo} object with the following structure:
|
|
133
133
|
*
|
|
134
134
|
* ```javascript
|
|
135
135
|
* {
|
|
@@ -141,8 +141,8 @@ export declare class SystemStatus {
|
|
|
141
141
|
* ```
|
|
142
142
|
*
|
|
143
143
|
* Where the `isSystemIdle` property is set to `false` if the system
|
|
144
|
-
* has been overloaded in the full history of the {@
|
|
145
|
-
* (which is configurable in the {@
|
|
144
|
+
* has been overloaded in the full history of the {@apilink Snapshotter}
|
|
145
|
+
* (which is configurable in the {@apilink Snapshotter}) and `true` otherwise.
|
|
146
146
|
*/
|
|
147
147
|
getHistoricalStatus(): SystemInfo;
|
|
148
148
|
/**
|
|
@@ -6,25 +6,25 @@ const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
|
6
6
|
const utils_1 = require("@crawlee/utils");
|
|
7
7
|
const snapshotter_1 = require("./snapshotter");
|
|
8
8
|
/**
|
|
9
|
-
* Provides a simple interface to reading system status from a {@
|
|
10
|
-
* It only exposes two functions {@
|
|
11
|
-
* and {@
|
|
9
|
+
* Provides a simple interface to reading system status from a {@apilink Snapshotter} instance.
|
|
10
|
+
* It only exposes two functions {@apilink SystemStatus.getCurrentStatus}
|
|
11
|
+
* and {@apilink SystemStatus.getHistoricalStatus}.
|
|
12
12
|
* The system status is calculated using a weighted average of overloaded
|
|
13
13
|
* messages in the snapshots, with the weights being the time intervals
|
|
14
14
|
* between the snapshots. Each resource is calculated separately
|
|
15
15
|
* and the system is overloaded whenever at least one resource is overloaded.
|
|
16
|
-
* The class is used by the {@
|
|
16
|
+
* The class is used by the {@apilink AutoscaledPool} class.
|
|
17
17
|
*
|
|
18
|
-
* {@
|
|
18
|
+
* {@apilink SystemStatus.getCurrentStatus}
|
|
19
19
|
* returns a boolean that represents the current status of the system.
|
|
20
20
|
* The length of the current timeframe in seconds is configurable
|
|
21
21
|
* by the `currentHistorySecs` option and represents the max age
|
|
22
22
|
* of snapshots to be considered for the calculation.
|
|
23
23
|
*
|
|
24
|
-
* {@
|
|
24
|
+
* {@apilink SystemStatus.getHistoricalStatus}
|
|
25
25
|
* returns a boolean that represents the long-term status
|
|
26
26
|
* of the system. It considers the full snapshot history available
|
|
27
|
-
* in the {@
|
|
27
|
+
* in the {@apilink Snapshotter} instance.
|
|
28
28
|
* @category Scaling
|
|
29
29
|
*/
|
|
30
30
|
class SystemStatus {
|
|
@@ -83,7 +83,7 @@ class SystemStatus {
|
|
|
83
83
|
this.snapshotter = snapshotter || new snapshotter_1.Snapshotter({ config });
|
|
84
84
|
}
|
|
85
85
|
/**
|
|
86
|
-
* Returns an {@
|
|
86
|
+
* Returns an {@apilink SystemInfo} object with the following structure:
|
|
87
87
|
*
|
|
88
88
|
* ```javascript
|
|
89
89
|
* {
|
|
@@ -102,7 +102,7 @@ class SystemStatus {
|
|
|
102
102
|
return this._isSystemIdle(this.currentHistorySecs);
|
|
103
103
|
}
|
|
104
104
|
/**
|
|
105
|
-
* Returns an {@
|
|
105
|
+
* Returns an {@apilink SystemInfo} object with the following structure:
|
|
106
106
|
*
|
|
107
107
|
* ```javascript
|
|
108
108
|
* {
|
|
@@ -114,8 +114,8 @@ class SystemStatus {
|
|
|
114
114
|
* ```
|
|
115
115
|
*
|
|
116
116
|
* Where the `isSystemIdle` property is set to `false` if the system
|
|
117
|
-
* has been overloaded in the full history of the {@
|
|
118
|
-
* (which is configurable in the {@
|
|
117
|
+
* has been overloaded in the full history of the {@apilink Snapshotter}
|
|
118
|
+
* (which is configurable in the {@apilink Snapshotter}) and `true` otherwise.
|
|
119
119
|
*/
|
|
120
120
|
getHistoricalStatus() {
|
|
121
121
|
return this._isSystemIdle();
|
package/configuration.d.ts
CHANGED
|
@@ -8,17 +8,17 @@ import type { EventManager } from './events';
|
|
|
8
8
|
export interface ConfigurationOptions {
|
|
9
9
|
/**
|
|
10
10
|
* Defines storage client to be used.
|
|
11
|
-
* @default {@
|
|
11
|
+
* @default {@apilink MemoryStorage}
|
|
12
12
|
*/
|
|
13
13
|
storageClient?: StorageClient;
|
|
14
14
|
/**
|
|
15
15
|
* Defines the Event Manager to be used.
|
|
16
|
-
* @default {@
|
|
16
|
+
* @default {@apilink EventManager}
|
|
17
17
|
*/
|
|
18
18
|
eventManager?: EventManager;
|
|
19
19
|
/**
|
|
20
20
|
* Could be used to adjust the storage client behavior
|
|
21
|
-
* e.g. {@
|
|
21
|
+
* e.g. {@apilink MemoryStorageOptions} could be used to adjust the {@apilink MemoryStorage} behavior.
|
|
22
22
|
*/
|
|
23
23
|
storageClientOptions?: Dictionary;
|
|
24
24
|
/**
|
|
@@ -56,7 +56,7 @@ export interface ConfigurationOptions {
|
|
|
56
56
|
*/
|
|
57
57
|
maxUsedCpuRatio?: number;
|
|
58
58
|
/**
|
|
59
|
-
* Sets the ratio, defining the amount of system memory that could be used by the {@
|
|
59
|
+
* Sets the ratio, defining the amount of system memory that could be used by the {@apilink AutoscaledPool}.
|
|
60
60
|
* When the memory usage is more than the provided ratio, the memory is considered overloaded.
|
|
61
61
|
*
|
|
62
62
|
* Alternative to `CRAWLEE_AVAILABLE_MEMORY_RATIO` environment variable.
|
|
@@ -64,7 +64,7 @@ export interface ConfigurationOptions {
|
|
|
64
64
|
*/
|
|
65
65
|
availableMemoryRatio?: number;
|
|
66
66
|
/**
|
|
67
|
-
* Sets the amount of system memory in megabytes to be used by the {@
|
|
67
|
+
* Sets the amount of system memory in megabytes to be used by the {@apilink AutoscaledPool}.
|
|
68
68
|
* By default, the maximum memory is set to one quarter of total system memory.
|
|
69
69
|
*
|
|
70
70
|
* Alternative to `CRAWLEE_MEMORY_MBYTES` environment variable.
|
|
@@ -84,7 +84,7 @@ export interface ConfigurationOptions {
|
|
|
84
84
|
systemInfoIntervalMillis?: number;
|
|
85
85
|
/**
|
|
86
86
|
* Defines the default input key, i.e. the key that is used to get the crawler input value
|
|
87
|
-
* from the default {@
|
|
87
|
+
* from the default {@apilink KeyValueStore} associated with the current crawler run.
|
|
88
88
|
*
|
|
89
89
|
* Alternative to `CRAWLEE_INPUT_KEY` environment variable.
|
|
90
90
|
* @default 'INPUT'
|
|
@@ -224,8 +224,8 @@ export declare class Configuration {
|
|
|
224
224
|
*/
|
|
225
225
|
set(key: keyof ConfigurationOptions, value?: any): void;
|
|
226
226
|
/**
|
|
227
|
-
* Returns cached instance of {@
|
|
228
|
-
* this {@
|
|
227
|
+
* Returns cached instance of {@apilink StorageClient} using options as defined in the environment variables or in
|
|
228
|
+
* this {@apilink Configuration} instance. Only first call of this method will create the client, following calls will
|
|
229
229
|
* return the same client instance.
|
|
230
230
|
*
|
|
231
231
|
* Caching works based on the `storageClientOptions`, so calling this method with different options will return
|
|
@@ -246,11 +246,11 @@ export declare class Configuration {
|
|
|
246
246
|
*/
|
|
247
247
|
static getGlobalConfig(): Configuration;
|
|
248
248
|
/**
|
|
249
|
-
* Gets default {@
|
|
249
|
+
* Gets default {@apilink StorageClient} instance.
|
|
250
250
|
*/
|
|
251
251
|
static getStorageClient(): StorageClient;
|
|
252
252
|
/**
|
|
253
|
-
* Gets default {@
|
|
253
|
+
* Gets default {@apilink EventManager} instance.
|
|
254
254
|
*/
|
|
255
255
|
static getEventManager(): EventManager;
|
|
256
256
|
/**
|
package/configuration.js
CHANGED
|
@@ -141,8 +141,8 @@ class Configuration {
|
|
|
141
141
|
this.options.set(key, value);
|
|
142
142
|
}
|
|
143
143
|
/**
|
|
144
|
-
* Returns cached instance of {@
|
|
145
|
-
* this {@
|
|
144
|
+
* Returns cached instance of {@apilink StorageClient} using options as defined in the environment variables or in
|
|
145
|
+
* this {@apilink Configuration} instance. Only first call of this method will create the client, following calls will
|
|
146
146
|
* return the same client instance.
|
|
147
147
|
*
|
|
148
148
|
* Caching works based on the `storageClientOptions`, so calling this method with different options will return
|
|
@@ -197,13 +197,13 @@ class Configuration {
|
|
|
197
197
|
return Configuration.globalConfig;
|
|
198
198
|
}
|
|
199
199
|
/**
|
|
200
|
-
* Gets default {@
|
|
200
|
+
* Gets default {@apilink StorageClient} instance.
|
|
201
201
|
*/
|
|
202
202
|
static getStorageClient() {
|
|
203
203
|
return this.getGlobalConfig().getStorageClient();
|
|
204
204
|
}
|
|
205
205
|
/**
|
|
206
|
-
* Gets default {@
|
|
206
|
+
* Gets default {@apilink EventManager} instance.
|
|
207
207
|
*/
|
|
208
208
|
static getEventManager() {
|
|
209
209
|
return this.getGlobalConfig().getEventManager();
|
|
@@ -6,13 +6,13 @@ import type { Session } from '../session_pool/session';
|
|
|
6
6
|
export interface CrawlingContext<UserData extends Dictionary = Dictionary> extends Record<PropertyKey, unknown> {
|
|
7
7
|
id: string;
|
|
8
8
|
/**
|
|
9
|
-
* The original {@
|
|
9
|
+
* The original {@apilink Request} object.
|
|
10
10
|
*/
|
|
11
11
|
request: Request<UserData>;
|
|
12
12
|
session?: Session;
|
|
13
13
|
/**
|
|
14
14
|
* An object with information about currently used proxy by the crawler
|
|
15
|
-
* and configured by the {@
|
|
15
|
+
* and configured by the {@apilink ProxyConfiguration} class.
|
|
16
16
|
*/
|
|
17
17
|
proxyInfo?: ProxyInfo;
|
|
18
18
|
log: Log;
|
package/crawlers/statistics.d.ts
CHANGED
|
@@ -28,7 +28,7 @@ export declare class Statistics {
|
|
|
28
28
|
*/
|
|
29
29
|
readonly id: number;
|
|
30
30
|
/**
|
|
31
|
-
* Current statistic state used for doing calculations on {@
|
|
31
|
+
* Current statistic state used for doing calculations on {@apilink Statistics.calculate} calls
|
|
32
32
|
*/
|
|
33
33
|
state: StatisticState;
|
|
34
34
|
/**
|
package/crawlers/statistics.js
CHANGED
|
@@ -67,7 +67,7 @@ class Statistics {
|
|
|
67
67
|
value: Statistics.id++
|
|
68
68
|
}); // assign an id while incrementing so it can be saved/restored from KV
|
|
69
69
|
/**
|
|
70
|
-
* Current statistic state used for doing calculations on {@
|
|
70
|
+
* Current statistic state used for doing calculations on {@apilink Statistics.calculate} calls
|
|
71
71
|
*/
|
|
72
72
|
Object.defineProperty(this, "state", {
|
|
73
73
|
enumerable: true,
|
|
@@ -10,9 +10,9 @@ export interface EnqueueLinksOptions {
|
|
|
10
10
|
requestQueue: RequestQueue;
|
|
11
11
|
/** A CSS selector matching links to be enqueued. */
|
|
12
12
|
selector?: string;
|
|
13
|
-
/** Sets {@
|
|
13
|
+
/** Sets {@apilink Request.userData} for newly enqueued requests. */
|
|
14
14
|
userData?: Dictionary;
|
|
15
|
-
/** Sets {@
|
|
15
|
+
/** Sets {@apilink Request.label} for newly enqueued requests. */
|
|
16
16
|
label?: string;
|
|
17
17
|
/**
|
|
18
18
|
* A base URL that will be used to resolve relative URLs when using Cheerio. Ignored when using Puppeteer,
|
|
@@ -24,7 +24,7 @@ export interface EnqueueLinksOptions {
|
|
|
24
24
|
* containing glob pattern strings matching the URLs to be enqueued.
|
|
25
25
|
*
|
|
26
26
|
* The plain objects must include at least the `glob` property, which holds the glob pattern string.
|
|
27
|
-
* All remaining keys will be used as request options for the corresponding enqueued {@
|
|
27
|
+
* All remaining keys will be used as request options for the corresponding enqueued {@apilink Request} objects.
|
|
28
28
|
*
|
|
29
29
|
* The matching is always case-insensitive.
|
|
30
30
|
* If you need case-sensitive matching, use `regexps` property directly.
|
|
@@ -38,7 +38,7 @@ export interface EnqueueLinksOptions {
|
|
|
38
38
|
* containing regular expressions matching the URLs to be enqueued.
|
|
39
39
|
*
|
|
40
40
|
* The plain objects must include at least the `regexp` property, which holds the regular expression.
|
|
41
|
-
* All remaining keys will be used as request options for the corresponding enqueued {@
|
|
41
|
+
* All remaining keys will be used as request options for the corresponding enqueued {@apilink Request} objects.
|
|
42
42
|
*
|
|
43
43
|
* If `regexps` is an empty array or `undefined`, and `globs` are also not defined, then the function
|
|
44
44
|
* enqueues the links with the same subdomain.
|
|
@@ -48,11 +48,11 @@ export interface EnqueueLinksOptions {
|
|
|
48
48
|
* *NOTE:* In future versions of SDK the options will be removed.
|
|
49
49
|
* Please use `globs` or `regexps` instead.
|
|
50
50
|
*
|
|
51
|
-
* An array of {@
|
|
52
|
-
* containing {@
|
|
51
|
+
* An array of {@apilink PseudoUrl} strings or plain objects
|
|
52
|
+
* containing {@apilink PseudoUrl} strings matching the URLs to be enqueued.
|
|
53
53
|
*
|
|
54
54
|
* The plain objects must include at least the `purl` property, which holds the pseudo-URL string.
|
|
55
|
-
* All remaining keys will be used as request options for the corresponding enqueued {@
|
|
55
|
+
* All remaining keys will be used as request options for the corresponding enqueued {@apilink Request} objects.
|
|
56
56
|
*
|
|
57
57
|
* With a pseudo-URL string, the matching is always case-insensitive.
|
|
58
58
|
* If you need case-sensitive matching, use `regexps` property directly.
|
|
@@ -64,7 +64,7 @@ export interface EnqueueLinksOptions {
|
|
|
64
64
|
*/
|
|
65
65
|
pseudoUrls?: PseudoUrlInput[];
|
|
66
66
|
/**
|
|
67
|
-
* Just before a new {@
|
|
67
|
+
* Just before a new {@apilink Request} is constructed and enqueued to the {@apilink RequestQueue}, this function can be used
|
|
68
68
|
* to remove it or modify its contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
|
|
69
69
|
* when you need to enqueue multiple `Requests` to the queue that share the same URL, but differ in methods or payloads,
|
|
70
70
|
* or to dynamically update or create `userData`.
|
|
@@ -113,11 +113,11 @@ export declare enum EnqueueStrategy {
|
|
|
113
113
|
SameDomain = "same-domain"
|
|
114
114
|
}
|
|
115
115
|
/**
|
|
116
|
-
* This function enqueues the urls provided to the {@
|
|
116
|
+
* This function enqueues the urls provided to the {@apilink RequestQueue} provided. If you want to automatically find and enqueue links,
|
|
117
117
|
* you should use the context-aware `enqueueLinks` function provided on the crawler contexts.
|
|
118
118
|
*
|
|
119
119
|
* Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
|
|
120
|
-
* and override settings of the enqueued {@
|
|
120
|
+
* and override settings of the enqueued {@apilink Request} objects.
|
|
121
121
|
*
|
|
122
122
|
* **Example usage**
|
|
123
123
|
*
|
|
@@ -134,14 +134,15 @@ export declare enum EnqueueStrategy {
|
|
|
134
134
|
* ```
|
|
135
135
|
*
|
|
136
136
|
* @param options All `enqueueLinks()` parameters are passed via an options object.
|
|
137
|
-
* @returns Promise that resolves to {@
|
|
137
|
+
* @returns Promise that resolves to {@apilink BatchAddRequestsResult} object.
|
|
138
138
|
*/
|
|
139
139
|
export declare function enqueueLinks(options: EnqueueLinksOptions): Promise<BatchAddRequestsResult>;
|
|
140
140
|
/**
|
|
141
141
|
* @internal
|
|
142
|
-
* This method helps resolve the baseUrl that will be used for filtering in {@
|
|
142
|
+
* This method helps resolve the baseUrl that will be used for filtering in {@apilink enqueueLinks}.
|
|
143
143
|
* - If a user provides a base url, we always return it
|
|
144
|
-
* - If a user specifies {@
|
|
144
|
+
* - If a user specifies {@apilink EnqueueStrategy.All} strategy, they do not care if the newly found urls are on the original
|
|
145
|
+
* request domain, or a redirected one
|
|
145
146
|
* - In all other cases, we return the domain of the original request as that's the one we need to use for filtering
|
|
146
147
|
*/
|
|
147
148
|
export declare function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalRequestUrl, originalRequestUrl, userProvidedBaseUrl, }: ResolveBaseUrl): string | undefined;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,gBAAgB,EAAoB,MAAM,UAAU,CAAC;AAQ3G,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAG9D,MAAM,WAAW,mBAAmB;IAChC,sHAAsH;IACtH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,EAAE,MAAM,EAAE,CAAC;IAEf,0DAA0D;IAC1D,YAAY,EAAE,YAAY,CAAC;IAE3B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,
|
|
1
|
+
{"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,gBAAgB,EAAoB,MAAM,UAAU,CAAC;AAQ3G,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAG9D,MAAM,WAAW,mBAAmB;IAChC,sHAAsH;IACtH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,EAAE,MAAM,EAAE,CAAC;IAEf,0DAA0D;IAC1D,YAAY,EAAE,YAAY,CAAC;IAE3B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,oEAAoE;IACpE,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB,iEAAiE;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IAEpB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IAExB;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU,CAAC,EAAE,cAAc,EAAE,CAAC;IAE9B;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;OAGG;IACH,QAAQ,CAAC,EAAE,eAAe,GAAG,KAAK,GAAG,aAAa,GAAG,eAAe,CAAC;CACxE;AAED,oBAAY,eAAe;IACvB;;OAEG;IACH,GAAG,QAAQ;IAEX;;;;OAIG;IACH,YAAY,kBAAkB;IAE9B;;;;OAIG;IACH,UAAU,gBAAgB;CAC7B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAuGhG;AAED;;;;;;;GAOG;AACH,wBAAgB,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACtB,EAAE,cAAc,sBA+BhB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,eAAe,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;IAClD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B"}
|
|
@@ -26,11 +26,11 @@ var EnqueueStrategy;
|
|
|
26
26
|
EnqueueStrategy["SameDomain"] = "same-domain";
|
|
27
27
|
})(EnqueueStrategy = exports.EnqueueStrategy || (exports.EnqueueStrategy = {}));
|
|
28
28
|
/**
|
|
29
|
-
* This function enqueues the urls provided to the {@
|
|
29
|
+
* This function enqueues the urls provided to the {@apilink RequestQueue} provided. If you want to automatically find and enqueue links,
|
|
30
30
|
* you should use the context-aware `enqueueLinks` function provided on the crawler contexts.
|
|
31
31
|
*
|
|
32
32
|
* Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
|
|
33
|
-
* and override settings of the enqueued {@
|
|
33
|
+
* and override settings of the enqueued {@apilink Request} objects.
|
|
34
34
|
*
|
|
35
35
|
* **Example usage**
|
|
36
36
|
*
|
|
@@ -47,7 +47,7 @@ var EnqueueStrategy;
|
|
|
47
47
|
* ```
|
|
48
48
|
*
|
|
49
49
|
* @param options All `enqueueLinks()` parameters are passed via an options object.
|
|
50
|
-
* @returns Promise that resolves to {@
|
|
50
|
+
* @returns Promise that resolves to {@apilink BatchAddRequestsResult} object.
|
|
51
51
|
*/
|
|
52
52
|
async function enqueueLinks(options) {
|
|
53
53
|
(0, ow_1.default)(options, ow_1.default.object.exactShape({
|
|
@@ -124,9 +124,10 @@ async function enqueueLinks(options) {
|
|
|
124
124
|
exports.enqueueLinks = enqueueLinks;
|
|
125
125
|
/**
|
|
126
126
|
* @internal
|
|
127
|
-
* This method helps resolve the baseUrl that will be used for filtering in {@
|
|
127
|
+
* This method helps resolve the baseUrl that will be used for filtering in {@apilink enqueueLinks}.
|
|
128
128
|
* - If a user provides a base url, we always return it
|
|
129
|
-
* - If a user specifies {@
|
|
129
|
+
* - If a user specifies {@apilink EnqueueStrategy.All} strategy, they do not care if the newly found urls are on the original
|
|
130
|
+
* request domain, or a redirected one
|
|
130
131
|
* - In all other cases, we return the domain of the original request as that's the one we need to use for filtering
|
|
131
132
|
*/
|
|
132
133
|
function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalRequestUrl, originalRequestUrl, userProvidedBaseUrl, }) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":";;;;AAAA,iCAAkC;AAClC,oDAAoB;AACpB,6DAA6B;AAG7B,qCAMkB;AA6GlB,IAAY,eAmBX;AAnBD,WAAY,eAAe;IACvB;;OAEG;IACH,8BAAW,CAAA;IAEX;;;;OAIG;IACH,iDAA8B,CAAA;IAE9B;;;;OAIG;IACH,6CAA0B,CAAA;AAC9B,CAAC,EAnBW,eAAe,GAAf,uBAAe,KAAf,uBAAe,QAmB1B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACI,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC3D,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,IAAI,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QAChC,YAAY,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC;QACjE,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACvC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAClC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACpC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAC9B,CAAC;QACF,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC9C,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC;YAC/B,eAAe,CAAC,GAAG;YACnB,eAAe,CAAC,YAAY;YAC5B,eAAe,CAAC,UAAU;SAC7B,CAAC;KACL,CAAC,CAAC,CAAC;IAEJ,MAAM,EACF,YAAY,EACZ,KAAK,EACL,IAAI,EACJ,UAAU,EACV,KAAK,EACL,OAAO,EACP,wBAAwB,GAC3B,GAAG,OAAO,CAAC;IAEZ,MAAM,iBAAiB,GAAuB,EAAE,CAAC;IAEjD,IAAI,UAAU,EAAE,MAAM,EAAE;QACpB,aAAG,CAAC,UAAU,CAAC,qEAAqE,CAAC,CAAC;QACtF,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,6CAAoC,EAAC,UAAU,CAAC,CAAC,CAAC;KAC/E;IAED,IAAI,KAAK,EAAE,MAAM,EAAE;QACf,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,sCAA6B,EAAC,KAAK,CAAC,CAAC,CAAC;KACnE;IAED,IAAI,OAAO,EAAE,MAAM,EAAE;QACjB,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,0CAAiC,EAAC,OAAO,CAAC,CAAC,CAAC;KACzE;IAED,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE;QAC3B,OAAO,CAAC,QAAQ,KAAhB,OAAO,CAAC,QAAQ,GAAK,eAAe,CAAC,YAAY,EAAC;KACrD;IAED,IAAI,OAAO,CAAC,OAAO,EAAE;QACjB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAErC,QAAQ,OAAO,CAAC,QAAQ,EAAE;YACtB,KAAK,eAAe,CAAC,YAAY;gBAC7B,sFAAsF;gBACtF,uFAAuF;gBACvF,yCAAyC;gBACzC,iBAAiB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;gBACrD,MAAM;YACV,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC;gBAC7B,4CAA4C;gBAC5C,MAAM,eAAe,GAAG,IAAA,iBAAS,EAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;gBAExE,IAAI,eAAe,EAAE;oBACjB,gHAAgH;oBAChH,GAAG,CAAC,QAAQ,GAAG,eAAe,CAAC;oBAC/B,iBAAiB,CAAC,IAAI,CAClB,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,KAAK,eAAe,EAAE,CAAC,KAAK,EAAE,EAC7E,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAC/B,CAAC;iBACL;qBAAM;oBACH,6FAA6F;oBAC7F,4BAA4B;oBAC5B,iBAAiB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;iBACxD;gBAED,MAAM;aACT;YACD,KAAK,eAAe,CAAC,GAAG,CAAC;YACzB;gBACI,MAAM;SACb;KACJ;IAED,IAAI,cAAc,GAAG,IAAA,6BAAoB,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAEzD,IAAI,wBAAwB,EAAE;QAC1B,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,wBAAwB,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAqB,CAAC;KAC9H;IAED,IAAI,QAAQ,GAAG,IAAA,uBAAc,EAAC,cAAc,EAAE,iBAAiB,CAAC,CAAC;IACjE,IAAI,KAAK;QAAE,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAE/C,OAAO,YAAY,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;AAC9C,CAAC;AAvGD,oCAuGC;AAED
|
|
1
|
+
{"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":";;;;AAAA,iCAAkC;AAClC,oDAAoB;AACpB,6DAA6B;AAG7B,qCAMkB;AA6GlB,IAAY,eAmBX;AAnBD,WAAY,eAAe;IACvB;;OAEG;IACH,8BAAW,CAAA;IAEX;;;;OAIG;IACH,iDAA8B,CAAA;IAE9B;;;;OAIG;IACH,6CAA0B,CAAA;AAC9B,CAAC,EAnBW,eAAe,GAAf,uBAAe,KAAf,uBAAe,QAmB1B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACI,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC3D,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,IAAI,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QAChC,YAAY,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC;QACjE,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACvC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAClC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACpC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAC9B,CAAC;QACF,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC9C,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC;YAC/B,eAAe,CAAC,GAAG;YACnB,eAAe,CAAC,YAAY;YAC5B,eAAe,CAAC,UAAU;SAC7B,CAAC;KACL,CAAC,CAAC,CAAC;IAEJ,MAAM,EACF,YAAY,EACZ,KAAK,EACL,IAAI,EACJ,UAAU,EACV,KAAK,EACL,OAAO,EACP,wBAAwB,GAC3B,GAAG,OAAO,CAAC;IAEZ,MAAM,iBAAiB,GAAuB,EAAE,CAAC;IAEjD,IAAI,UAAU,EAAE,MAAM,EAAE;QACpB,aAAG,CAAC,UAAU,CAAC,qEAAqE,CAAC,CAAC;QACtF,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,6CAAoC,EAAC,UAAU,CAAC,CAAC,CAAC;KAC/E;IAED,IAAI,KAAK,EAAE,MAAM,EAAE;QACf,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,sCAA6B,EAAC,KAAK,CAAC,CAAC,CAAC;KACnE;IAED,IAAI,OAAO,EAAE,MAAM,EAAE;QACjB,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,0CAAiC,EAAC,OAAO,CAAC,CAAC,CAAC;KACzE;IAED,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE;QAC3B,OAAO,CAAC,QAAQ,KAAhB,OAAO,CAAC,QAAQ,GAAK,eAAe,CAAC,YAAY,EAAC;KACrD;IAED,IAAI,OAAO,CAAC,OAAO,EAAE;QACjB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAErC,QAAQ,OAAO,CAAC,QAAQ,EAAE;YACtB,KAAK,eAAe,CAAC,YAAY;gBAC7B,sFAAsF;gBACtF,uFAAuF;gBACvF,yCAAyC;gBACzC,iBAAiB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;gBACrD,MAAM;YACV,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC;gBAC7B,4CAA4C;gBAC5C,MAAM,eAAe,GAAG,IAAA,iBAAS,EAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;gBAExE,IAAI,eAAe,EAAE;oBACjB,gHAAgH;oBAChH,GAAG,CAAC,QAAQ,GAAG,eAAe,CAAC;oBAC/B,iBAAiB,CAAC,IAAI,CAClB,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,KAAK,eAAe,EAAE,CAAC,KAAK,EAAE,EAC7E,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAC/B,CAAC;iBACL;qBAAM;oBACH,6FAA6F;oBAC7F,4BAA4B;oBAC5B,iBAAiB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;iBACxD;gBAED,MAAM;aACT;YACD,KAAK,eAAe,CAAC,GAAG,CAAC;YACzB;gBACI,MAAM;SACb;KACJ;IAED,IAAI,cAAc,GAAG,IAAA,6BAAoB,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAEzD,IAAI,wBAAwB,EAAE;QAC1B,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,wBAAwB,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAqB,CAAC;KAC9H;IAED,IAAI,QAAQ,GAAG,IAAA,uBAAc,EAAC,cAAc,EAAE,iBAAiB,CAAC,CAAC;IACjE,IAAI,KAAK;QAAE,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAE/C,OAAO,YAAY,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;AAC9C,CAAC;AAvGD,oCAuGC;AAED;;;;;;;GAOG;AACH,SAAgB,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACN;IACb,wCAAwC;IACxC,IAAI,mBAAmB,EAAE;QACrB,OAAO,mBAAmB,CAAC;KAC9B;IAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAC7D,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,eAAe,IAAI,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAE7E,6DAA6D;IAC7D,IAAI,eAAe,KAAK,eAAe,CAAC,GAAG,EAAE;QACzC,OAAO,cAAc,CAAC;KACzB;IAED,0HAA0H;IAC1H,+IAA+I;IAC/I,4DAA4D;IAC5D,IAAI,eAAe,KAAK,eAAe,CAAC,UAAU,EAAE;QAChD,MAAM,gBAAgB,GAAG,IAAA,iBAAS,EAAC,iBAAiB,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAC/E,MAAM,aAAa,GAAG,IAAA,iBAAS,EAAC,cAAc,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAEzE,IAAI,gBAAgB,KAAK,aAAa,EAAE;YACpC,OAAO,cAAc,CAAC;SACzB;QAED,OAAO,SAAS,CAAC;KACpB;IAED,2JAA2J;IAC3J,mCAAmC;IACnC,OAAO,iBAAiB,CAAC;AAC7B,CAAC;AApCD,wFAoCC"}
|
|
@@ -52,8 +52,8 @@ export declare function createRequests(requestOptions: (string | RequestOptions)
|
|
|
52
52
|
*/
|
|
53
53
|
export declare function createRequestOptions(sources: (string | Record<string, unknown>)[], options?: Pick<EnqueueLinksOptions, 'label' | 'userData'>): RequestOptions[];
|
|
54
54
|
/**
|
|
55
|
-
* Takes an Apify {@
|
|
56
|
-
* {@
|
|
55
|
+
* Takes an Apify {@apilink RequestOptions} object and changes its attributes in a desired way. This user-function is used
|
|
56
|
+
* {@apilink enqueueLinks} to modify requests before enqueuing them.
|
|
57
57
|
*/
|
|
58
58
|
export interface RequestTransform {
|
|
59
59
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "3.0.3-beta.
|
|
3
|
+
"version": "3.0.3-beta.11",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -58,9 +58,9 @@
|
|
|
58
58
|
"@apify/pseudo_url": "^2.0.0",
|
|
59
59
|
"@apify/timeout": "^0.3.0",
|
|
60
60
|
"@apify/utilities": "^2.0.0",
|
|
61
|
-
"@crawlee/memory-storage": "^3.0.3-beta.
|
|
62
|
-
"@crawlee/types": "^3.0.3-beta.
|
|
63
|
-
"@crawlee/utils": "^3.0.3-beta.
|
|
61
|
+
"@crawlee/memory-storage": "^3.0.3-beta.11",
|
|
62
|
+
"@crawlee/types": "^3.0.3-beta.11",
|
|
63
|
+
"@crawlee/utils": "^3.0.3-beta.11",
|
|
64
64
|
"@types/tough-cookie": "^4.0.2",
|
|
65
65
|
"@vladfrangu/async_event_emitter": "^2.0.0",
|
|
66
66
|
"fs-extra": "^10.1.0",
|