@crawlee/core 3.0.3-beta.7 → 3.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/autoscaling/autoscaled_pool.d.ts +16 -16
- package/autoscaling/autoscaled_pool.js +13 -13
- package/autoscaling/snapshotter.d.ts +1 -1
- package/autoscaling/snapshotter.js +1 -1
- package/autoscaling/system_status.d.ts +12 -12
- package/autoscaling/system_status.js +11 -11
- package/configuration.d.ts +33 -13
- package/configuration.d.ts.map +1 -1
- package/configuration.js +27 -7
- package/configuration.js.map +1 -1
- package/crawlers/crawler_commons.d.ts +2 -2
- package/crawlers/statistics.d.ts +1 -1
- package/crawlers/statistics.js +1 -1
- package/enqueue_links/enqueue_links.d.ts +14 -13
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +6 -5
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +2 -2
- package/errors.d.ts +6 -0
- package/errors.d.ts.map +1 -1
- package/errors.js +10 -1
- package/errors.js.map +1 -1
- package/index.mjs +1 -0
- package/package.json +4 -4
- package/proxy_configuration.d.ts +11 -11
- package/proxy_configuration.js +8 -8
- package/request.d.ts +3 -3
- package/request.js +2 -2
- package/session_pool/session.d.ts +1 -1
- package/session_pool/session_pool.d.ts +12 -12
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +10 -10
- package/storages/dataset.d.ts +14 -14
- package/storages/dataset.js +8 -8
- package/storages/key_value_store.d.ts +30 -49
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +51 -46
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts +33 -33
- package/storages/request_list.js +23 -23
- package/storages/request_queue.d.ts +25 -25
- package/storages/request_queue.js +23 -23
- package/storages/utils.d.ts +1 -1
- package/storages/utils.js +1 -1
- package/tsconfig.build.tsbuildinfo +1 -1
|
@@ -9,8 +9,8 @@ export declare const STATE_PERSISTENCE_KEY = "REQUEST_LIST_STATE";
|
|
|
9
9
|
export declare const REQUESTS_PERSISTENCE_KEY = "REQUEST_LIST_REQUESTS";
|
|
10
10
|
export interface RequestListOptions {
|
|
11
11
|
/**
|
|
12
|
-
* An array of sources of URLs for the {@
|
|
13
|
-
* plain objects that define at least the `url` property, or an array of {@
|
|
12
|
+
* An array of sources of URLs for the {@apilink RequestList}. It can be either an array of strings,
|
|
13
|
+
* plain objects that define at least the `url` property, or an array of {@apilink Request} instances.
|
|
14
14
|
*
|
|
15
15
|
* **IMPORTANT:** The `sources` array will be consumed (left empty) after `RequestList` initializes.
|
|
16
16
|
* This is a measure to prevent memory leaks in situations when millions of sources are
|
|
@@ -44,15 +44,15 @@ export interface RequestListOptions {
|
|
|
44
44
|
sources?: Source[];
|
|
45
45
|
/**
|
|
46
46
|
* A function that will be called to get the sources for the `RequestList`, but only if `RequestList`
|
|
47
|
-
* was not able to fetch their persisted version (see {@
|
|
48
|
-
* It must return an `Array` of {@
|
|
47
|
+
* was not able to fetch their persisted version (see {@apilink RequestListOptions.persistRequestsKey}).
|
|
48
|
+
* It must return an `Array` of {@apilink Request} or {@apilink RequestOptions}.
|
|
49
49
|
*
|
|
50
50
|
* This is very useful in a scenario when getting the sources is a resource intensive or time consuming
|
|
51
51
|
* task, such as fetching URLs from multiple sitemaps or parsing URLs from large datasets. Using the
|
|
52
52
|
* `sourcesFunction` in combination with `persistStateKey` and `persistRequestsKey` will allow you to
|
|
53
53
|
* fetch and parse those URLs only once, saving valuable time when your crawler migrates or restarts.
|
|
54
54
|
*
|
|
55
|
-
* If both {@
|
|
55
|
+
* If both {@apilink RequestListOptions.sources} and {@apilink RequestListOptions.sourcesFunction} are provided,
|
|
56
56
|
* the sources returned by the function will be added after the `sources`.
|
|
57
57
|
*
|
|
58
58
|
* **Example:**
|
|
@@ -99,12 +99,12 @@ export interface RequestListOptions {
|
|
|
99
99
|
persistStateKey?: string;
|
|
100
100
|
/**
|
|
101
101
|
* Identifies the key in the default key-value store under which the `RequestList` persists its
|
|
102
|
-
* Requests during the {@
|
|
102
|
+
* Requests during the {@apilink RequestList.initialize} call.
|
|
103
103
|
* This is necessary if `persistStateKey` is set and the source URLs might potentially change,
|
|
104
104
|
* to ensure consistency of the source URLs and state object. However, it comes with some
|
|
105
105
|
* storage and performance overheads.
|
|
106
106
|
*
|
|
107
|
-
* If `persistRequestsKey` is not set, {@
|
|
107
|
+
* If `persistRequestsKey` is not set, {@apilink RequestList.initialize} will always fetch the sources
|
|
108
108
|
* from their origin, check that they are consistent with the restored state (if any)
|
|
109
109
|
* and throw an error if they are not.
|
|
110
110
|
*/
|
|
@@ -130,7 +130,7 @@ export interface RequestListOptions {
|
|
|
130
130
|
state?: RequestListState;
|
|
131
131
|
/**
|
|
132
132
|
* By default, `RequestList` will deduplicate the provided URLs. Default deduplication is based
|
|
133
|
-
* on the `uniqueKey` property of passed source {@
|
|
133
|
+
* on the `uniqueKey` property of passed source {@apilink Request} objects.
|
|
134
134
|
*
|
|
135
135
|
* If the property is not present, it is generated by normalizing the URL. If present, it is kept intact.
|
|
136
136
|
* In any case, only one request per `uniqueKey` is added to the `RequestList` resulting in removal
|
|
@@ -150,19 +150,19 @@ export interface RequestListOptions {
|
|
|
150
150
|
/**
|
|
151
151
|
* Represents a static list of URLs to crawl.
|
|
152
152
|
* The URLs can be provided either in code or parsed from a text file hosted on the web.
|
|
153
|
-
* `RequestList` is used by {@
|
|
154
|
-
* and {@
|
|
153
|
+
* `RequestList` is used by {@apilink BasicCrawler}, {@apilink CheerioCrawler}, {@apilink PuppeteerCrawler}
|
|
154
|
+
* and {@apilink PlaywrightCrawler} as a source of URLs to crawl.
|
|
155
155
|
*
|
|
156
|
-
* Each URL is represented using an instance of the {@
|
|
156
|
+
* Each URL is represented using an instance of the {@apilink Request} class.
|
|
157
157
|
* The list can only contain unique URLs. More precisely, it can only contain `Request` instances
|
|
158
158
|
* with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
|
|
159
|
-
* To add a single URL to the list multiple times, corresponding {@
|
|
159
|
+
* To add a single URL to the list multiple times, corresponding {@apilink Request} objects will need to have different
|
|
160
160
|
* `uniqueKey` properties. You can use the `keepDuplicateUrls` option to do this for you when initializing the
|
|
161
161
|
* `RequestList` from sources.
|
|
162
162
|
*
|
|
163
|
-
* Once you create an instance of `RequestList`, you need to call the {@
|
|
163
|
+
* Once you create an instance of `RequestList`, you need to call the {@apilink RequestList.initialize} function
|
|
164
164
|
* before the instance can be used. After that, no more URLs can be added to the list.
|
|
165
|
-
* Unlike {@
|
|
165
|
+
* Unlike {@apilink RequestQueue}, `RequestList` is static but it can contain even millions of URLs.
|
|
166
166
|
* > Note that `RequestList` can be used together with `RequestQueue` by the same crawler.
|
|
167
167
|
* > In such cases, each request from `RequestList` is enqueued into `RequestQueue` first and then consumed from the latter.
|
|
168
168
|
* > This is necessary to avoid the same URL being processed more than once (from the list first and then possibly from the queue).
|
|
@@ -171,9 +171,9 @@ export interface RequestListOptions {
|
|
|
171
171
|
*
|
|
172
172
|
* `RequestList` has an internal state where it stores information about which requests were already handled,
|
|
173
173
|
* which are in progress and which were reclaimed. The state may be automatically persisted to the default
|
|
174
|
-
* {@
|
|
174
|
+
* {@apilink KeyValueStore} by setting the `persistStateKey` option so that if the Node.js process is restarted,
|
|
175
175
|
* the crawling can continue where it left off. The automated persisting is launched upon receiving the `persistState`
|
|
176
|
-
* event that is periodically emitted by {@
|
|
176
|
+
* event that is periodically emitted by {@apilink EventManager}.
|
|
177
177
|
*
|
|
178
178
|
* The internal state is closely tied to the provided sources (URLs). If the sources change on crawler restart, the state will become corrupted and
|
|
179
179
|
* `RequestList` will raise an exception. This typically happens when the sources is a list of URLs downloaded from the web.
|
|
@@ -279,7 +279,7 @@ export declare class RequestList {
|
|
|
279
279
|
*/
|
|
280
280
|
protected _addRequestsFromSources(): Promise<void>;
|
|
281
281
|
/**
|
|
282
|
-
* Persists the current state of the `RequestList` into the default {@
|
|
282
|
+
* Persists the current state of the `RequestList` into the default {@apilink KeyValueStore}.
|
|
283
283
|
* The state is persisted automatically in regular intervals, but calling this method manually
|
|
284
284
|
* is useful in cases where you want to have the most current state available after you pause
|
|
285
285
|
* or stop fetching its requests. For example after you pause or abort a crawl. Or just before
|
|
@@ -307,7 +307,7 @@ export declare class RequestList {
|
|
|
307
307
|
*/
|
|
308
308
|
getState(): RequestListState;
|
|
309
309
|
/**
|
|
310
|
-
* Resolves to `true` if the next call to {@
|
|
310
|
+
* Resolves to `true` if the next call to {@apilink RequestList.fetchNextRequest} function
|
|
311
311
|
* would return `null`, otherwise it resolves to `false`.
|
|
312
312
|
* Note that even if the list is empty, there might be some pending requests currently being processed.
|
|
313
313
|
*/
|
|
@@ -317,8 +317,8 @@ export declare class RequestList {
|
|
|
317
317
|
*/
|
|
318
318
|
isFinished(): Promise<boolean>;
|
|
319
319
|
/**
|
|
320
|
-
* Gets the next {@
|
|
321
|
-
* using the {@
|
|
320
|
+
* Gets the next {@apilink Request} to process. First, the function gets a request previously reclaimed
|
|
321
|
+
* using the {@apilink RequestList.reclaimRequest} function, if there is any.
|
|
322
322
|
* Otherwise it gets the next request from sources.
|
|
323
323
|
*
|
|
324
324
|
* The function's `Promise` resolves to `null` if there are no more
|
|
@@ -372,14 +372,14 @@ export declare class RequestList {
|
|
|
372
372
|
handledCount(): number;
|
|
373
373
|
/**
|
|
374
374
|
* Opens a request list and returns a promise resolving to an instance
|
|
375
|
-
* of the {@
|
|
375
|
+
* of the {@apilink RequestList} class that is already initialized.
|
|
376
376
|
*
|
|
377
|
-
* {@
|
|
377
|
+
* {@apilink RequestList} represents a list of URLs to crawl, which is always stored in memory.
|
|
378
378
|
* To enable picking up where left off after a process restart, the request list sources
|
|
379
379
|
* are persisted to the key-value store at initialization of the list. Then, while crawling,
|
|
380
380
|
* a small state object is regularly persisted to keep track of the crawling status.
|
|
381
381
|
*
|
|
382
|
-
* For more details and code examples, see the {@
|
|
382
|
+
* For more details and code examples, see the {@apilink RequestList} class.
|
|
383
383
|
*
|
|
384
384
|
* **Example usage:**
|
|
385
385
|
*
|
|
@@ -405,23 +405,23 @@ export declare class RequestList {
|
|
|
405
405
|
* If `null`, the list will not be persisted and will only be stored in memory. Process restart
|
|
406
406
|
* will then cause the list to be crawled again from the beginning. We suggest always using a name.
|
|
407
407
|
* @param [sources]
|
|
408
|
-
* An array of sources of URLs for the {@
|
|
409
|
-
* plain objects that define at least the `url` property, or an array of {@
|
|
408
|
+
* An array of sources of URLs for the {@apilink RequestList}. It can be either an array of strings,
|
|
409
|
+
* plain objects that define at least the `url` property, or an array of {@apilink Request} instances.
|
|
410
410
|
*
|
|
411
|
-
* **IMPORTANT:** The `sources` array will be consumed (left empty) after {@
|
|
411
|
+
* **IMPORTANT:** The `sources` array will be consumed (left empty) after {@apilink RequestList} initializes.
|
|
412
412
|
* This is a measure to prevent memory leaks in situations when millions of sources are
|
|
413
413
|
* added.
|
|
414
414
|
*
|
|
415
415
|
* Additionally, the `requestsFromUrl` property may be used instead of `url`,
|
|
416
|
-
* which will instruct {@
|
|
416
|
+
* which will instruct {@apilink RequestList} to download the source URLs from a given remote location.
|
|
417
417
|
* The URLs will be parsed from the received response. In this case you can limit the URLs
|
|
418
418
|
* using `regex` parameter containing regular expression pattern for URLs to be included.
|
|
419
419
|
*
|
|
420
|
-
* For details, see the {@
|
|
420
|
+
* For details, see the {@apilink RequestListOptions.sources}
|
|
421
421
|
* @param [options]
|
|
422
|
-
* The {@
|
|
423
|
-
* the {@
|
|
424
|
-
* options and the `sources` parameter supersedes the {@
|
|
422
|
+
* The {@apilink RequestList} options. Note that the `listName` parameter supersedes
|
|
423
|
+
* the {@apilink RequestListOptions.persistStateKey} and {@apilink RequestListOptions.persistRequestsKey}
|
|
424
|
+
* options and the `sources` parameter supersedes the {@apilink RequestListOptions.sources} option.
|
|
425
425
|
*/
|
|
426
426
|
static open(listNameOrOptions: string | null | RequestListOptions, sources?: Source[], options?: RequestListOptions): Promise<RequestList>;
|
|
427
427
|
/**
|
|
@@ -430,8 +430,8 @@ export declare class RequestList {
|
|
|
430
430
|
private _downloadListOfUrls;
|
|
431
431
|
}
|
|
432
432
|
/**
|
|
433
|
-
* Represents state of a {@
|
|
434
|
-
* You can obtain the state by calling {@
|
|
433
|
+
* Represents state of a {@apilink RequestList}. It can be used to resume a {@apilink RequestList} which has been previously processed.
|
|
434
|
+
* You can obtain the state by calling {@apilink RequestList.getState} and receive an object with
|
|
435
435
|
* the following structure:
|
|
436
436
|
*
|
|
437
437
|
* ```
|
package/storages/request_list.js
CHANGED
|
@@ -18,19 +18,19 @@ const CONTENT_TYPE_BINARY = 'application/octet-stream';
|
|
|
18
18
|
/**
|
|
19
19
|
* Represents a static list of URLs to crawl.
|
|
20
20
|
* The URLs can be provided either in code or parsed from a text file hosted on the web.
|
|
21
|
-
* `RequestList` is used by {@
|
|
22
|
-
* and {@
|
|
21
|
+
* `RequestList` is used by {@apilink BasicCrawler}, {@apilink CheerioCrawler}, {@apilink PuppeteerCrawler}
|
|
22
|
+
* and {@apilink PlaywrightCrawler} as a source of URLs to crawl.
|
|
23
23
|
*
|
|
24
|
-
* Each URL is represented using an instance of the {@
|
|
24
|
+
* Each URL is represented using an instance of the {@apilink Request} class.
|
|
25
25
|
* The list can only contain unique URLs. More precisely, it can only contain `Request` instances
|
|
26
26
|
* with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
|
|
27
|
-
* To add a single URL to the list multiple times, corresponding {@
|
|
27
|
+
* To add a single URL to the list multiple times, corresponding {@apilink Request} objects will need to have different
|
|
28
28
|
* `uniqueKey` properties. You can use the `keepDuplicateUrls` option to do this for you when initializing the
|
|
29
29
|
* `RequestList` from sources.
|
|
30
30
|
*
|
|
31
|
-
* Once you create an instance of `RequestList`, you need to call the {@
|
|
31
|
+
* Once you create an instance of `RequestList`, you need to call the {@apilink RequestList.initialize} function
|
|
32
32
|
* before the instance can be used. After that, no more URLs can be added to the list.
|
|
33
|
-
* Unlike {@
|
|
33
|
+
* Unlike {@apilink RequestQueue}, `RequestList` is static but it can contain even millions of URLs.
|
|
34
34
|
* > Note that `RequestList` can be used together with `RequestQueue` by the same crawler.
|
|
35
35
|
* > In such cases, each request from `RequestList` is enqueued into `RequestQueue` first and then consumed from the latter.
|
|
36
36
|
* > This is necessary to avoid the same URL being processed more than once (from the list first and then possibly from the queue).
|
|
@@ -39,9 +39,9 @@ const CONTENT_TYPE_BINARY = 'application/octet-stream';
|
|
|
39
39
|
*
|
|
40
40
|
* `RequestList` has an internal state where it stores information about which requests were already handled,
|
|
41
41
|
* which are in progress and which were reclaimed. The state may be automatically persisted to the default
|
|
42
|
-
* {@
|
|
42
|
+
* {@apilink KeyValueStore} by setting the `persistStateKey` option so that if the Node.js process is restarted,
|
|
43
43
|
* the crawling can continue where it left off. The automated persisting is launched upon receiving the `persistState`
|
|
44
|
-
* event that is periodically emitted by {@
|
|
44
|
+
* event that is periodically emitted by {@apilink EventManager}.
|
|
45
45
|
*
|
|
46
46
|
* The internal state is closely tied to the provided sources (URLs). If the sources change on crawler restart, the state will become corrupted and
|
|
47
47
|
* `RequestList` will raise an exception. This typically happens when the sources is a list of URLs downloaded from the web.
|
|
@@ -338,7 +338,7 @@ class RequestList {
|
|
|
338
338
|
}
|
|
339
339
|
}
|
|
340
340
|
/**
|
|
341
|
-
* Persists the current state of the `RequestList` into the default {@
|
|
341
|
+
* Persists the current state of the `RequestList` into the default {@apilink KeyValueStore}.
|
|
342
342
|
* The state is persisted automatically in regular intervals, but calling this method manually
|
|
343
343
|
* is useful in cases where you want to have the most current state available after you pause
|
|
344
344
|
* or stop fetching its requests. For example after you pause or abort a crawl. Or just before
|
|
@@ -463,7 +463,7 @@ class RequestList {
|
|
|
463
463
|
};
|
|
464
464
|
}
|
|
465
465
|
/**
|
|
466
|
-
* Resolves to `true` if the next call to {@
|
|
466
|
+
* Resolves to `true` if the next call to {@apilink RequestList.fetchNextRequest} function
|
|
467
467
|
* would return `null`, otherwise it resolves to `false`.
|
|
468
468
|
* Note that even if the list is empty, there might be some pending requests currently being processed.
|
|
469
469
|
*/
|
|
@@ -479,8 +479,8 @@ class RequestList {
|
|
|
479
479
|
return this.inProgress.size === 0 && this.nextIndex >= this.requests.length;
|
|
480
480
|
}
|
|
481
481
|
/**
|
|
482
|
-
* Gets the next {@
|
|
483
|
-
* using the {@
|
|
482
|
+
* Gets the next {@apilink Request} to process. First, the function gets a request previously reclaimed
|
|
483
|
+
* using the {@apilink RequestList.reclaimRequest} function, if there is any.
|
|
484
484
|
* Otherwise it gets the next request from sources.
|
|
485
485
|
*
|
|
486
486
|
* The function's `Promise` resolves to `null` if there are no more
|
|
@@ -650,14 +650,14 @@ class RequestList {
|
|
|
650
650
|
}
|
|
651
651
|
/**
|
|
652
652
|
* Opens a request list and returns a promise resolving to an instance
|
|
653
|
-
* of the {@
|
|
653
|
+
* of the {@apilink RequestList} class that is already initialized.
|
|
654
654
|
*
|
|
655
|
-
* {@
|
|
655
|
+
* {@apilink RequestList} represents a list of URLs to crawl, which is always stored in memory.
|
|
656
656
|
* To enable picking up where left off after a process restart, the request list sources
|
|
657
657
|
* are persisted to the key-value store at initialization of the list. Then, while crawling,
|
|
658
658
|
* a small state object is regularly persisted to keep track of the crawling status.
|
|
659
659
|
*
|
|
660
|
-
* For more details and code examples, see the {@
|
|
660
|
+
* For more details and code examples, see the {@apilink RequestList} class.
|
|
661
661
|
*
|
|
662
662
|
* **Example usage:**
|
|
663
663
|
*
|
|
@@ -683,23 +683,23 @@ class RequestList {
|
|
|
683
683
|
* If `null`, the list will not be persisted and will only be stored in memory. Process restart
|
|
684
684
|
* will then cause the list to be crawled again from the beginning. We suggest always using a name.
|
|
685
685
|
* @param [sources]
|
|
686
|
-
* An array of sources of URLs for the {@
|
|
687
|
-
* plain objects that define at least the `url` property, or an array of {@
|
|
686
|
+
* An array of sources of URLs for the {@apilink RequestList}. It can be either an array of strings,
|
|
687
|
+
* plain objects that define at least the `url` property, or an array of {@apilink Request} instances.
|
|
688
688
|
*
|
|
689
|
-
* **IMPORTANT:** The `sources` array will be consumed (left empty) after {@
|
|
689
|
+
* **IMPORTANT:** The `sources` array will be consumed (left empty) after {@apilink RequestList} initializes.
|
|
690
690
|
* This is a measure to prevent memory leaks in situations when millions of sources are
|
|
691
691
|
* added.
|
|
692
692
|
*
|
|
693
693
|
* Additionally, the `requestsFromUrl` property may be used instead of `url`,
|
|
694
|
-
* which will instruct {@
|
|
694
|
+
* which will instruct {@apilink RequestList} to download the source URLs from a given remote location.
|
|
695
695
|
* The URLs will be parsed from the received response. In this case you can limit the URLs
|
|
696
696
|
* using `regex` parameter containing regular expression pattern for URLs to be included.
|
|
697
697
|
*
|
|
698
|
-
* For details, see the {@
|
|
698
|
+
* For details, see the {@apilink RequestListOptions.sources}
|
|
699
699
|
* @param [options]
|
|
700
|
-
* The {@
|
|
701
|
-
* the {@
|
|
702
|
-
* options and the `sources` parameter supersedes the {@
|
|
700
|
+
* The {@apilink RequestList} options. Note that the `listName` parameter supersedes
|
|
701
|
+
* the {@apilink RequestListOptions.persistStateKey} and {@apilink RequestListOptions.persistRequestsKey}
|
|
702
|
+
* options and the `sources` parameter supersedes the {@apilink RequestListOptions.sources} option.
|
|
703
703
|
*/
|
|
704
704
|
static async open(listNameOrOptions, sources, options = {}) {
|
|
705
705
|
if (listNameOrOptions != null && typeof listNameOrOptions === 'object') {
|
|
@@ -40,7 +40,7 @@ export declare const STORAGE_CONSISTENCY_DELAY_MILLIS = 3000;
|
|
|
40
40
|
export declare function getRequestId(uniqueKey: string): string;
|
|
41
41
|
/**
|
|
42
42
|
* A helper class that is used to report results from various
|
|
43
|
-
* {@
|
|
43
|
+
* {@apilink RequestQueue} functions as well as {@apilink enqueueLinks}.
|
|
44
44
|
*/
|
|
45
45
|
export interface QueueOperationInfo {
|
|
46
46
|
/** Indicates if request was already present in the queue. */
|
|
@@ -56,7 +56,7 @@ export interface RequestQueueOperationOptions {
|
|
|
56
56
|
* If set to `true`:
|
|
57
57
|
* - while adding the request to the queue: the request will be added to the foremost position in the queue.
|
|
58
58
|
* - while reclaiming the request: the request will be placed to the beginning of the queue, so that it's returned
|
|
59
|
-
* in the next call to {@
|
|
59
|
+
* in the next call to {@apilink RequestQueue.fetchNextRequest}.
|
|
60
60
|
* By default, it's put to the end of the queue.
|
|
61
61
|
* @default false
|
|
62
62
|
*/
|
|
@@ -67,17 +67,17 @@ export interface RequestQueueOperationOptions {
|
|
|
67
67
|
* where you start with several URLs and then recursively
|
|
68
68
|
* follow links to other pages. The data structure supports both breadth-first and depth-first crawling orders.
|
|
69
69
|
*
|
|
70
|
-
* Each URL is represented using an instance of the {@
|
|
71
|
-
* The queue can only contain unique URLs. More precisely, it can only contain {@
|
|
70
|
+
* Each URL is represented using an instance of the {@apilink Request} class.
|
|
71
|
+
* The queue can only contain unique URLs. More precisely, it can only contain {@apilink Request} instances
|
|
72
72
|
* with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
|
|
73
73
|
* To add a single URL multiple times to the queue,
|
|
74
|
-
* corresponding {@
|
|
74
|
+
* corresponding {@apilink Request} objects will need to have different `uniqueKey` properties.
|
|
75
75
|
*
|
|
76
|
-
* Do not instantiate this class directly, use the {@
|
|
76
|
+
* Do not instantiate this class directly, use the {@apilink RequestQueue.open} function instead.
|
|
77
77
|
*
|
|
78
|
-
* `RequestQueue` is used by {@
|
|
79
|
-
* and {@
|
|
80
|
-
* Unlike {@
|
|
78
|
+
* `RequestQueue` is used by {@apilink BasicCrawler}, {@apilink CheerioCrawler}, {@apilink PuppeteerCrawler}
|
|
79
|
+
* and {@apilink PlaywrightCrawler} as a source of URLs to crawl.
|
|
80
|
+
* Unlike {@apilink RequestList}, `RequestQueue` supports dynamic adding and removing of requests.
|
|
81
81
|
* On the other hand, the queue is not optimized for operations that add or remove a large number of URLs in a batch.
|
|
82
82
|
*
|
|
83
83
|
* `RequestQueue` stores its data either on local disk or in the Apify Cloud,
|
|
@@ -89,7 +89,7 @@ export interface RequestQueueOperationOptions {
|
|
|
89
89
|
* If the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the
|
|
90
90
|
* [Apify Request Queue](https://docs.apify.com/storage/request-queue)
|
|
91
91
|
* cloud storage. Note that you can force usage of the cloud storage also by passing the `forceCloud`
|
|
92
|
-
* option to {@
|
|
92
|
+
* option to {@apilink RequestQueue.open} function,
|
|
93
93
|
* even if the `APIFY_LOCAL_STORAGE_DIR` variable is set.
|
|
94
94
|
*
|
|
95
95
|
* **Example usage:**
|
|
@@ -152,12 +152,12 @@ export declare class RequestQueue {
|
|
|
152
152
|
*
|
|
153
153
|
* If a request with the same `uniqueKey` property is already present in the queue,
|
|
154
154
|
* it will not be updated. You can find out whether this happened from the resulting
|
|
155
|
-
* {@
|
|
155
|
+
* {@apilink QueueOperationInfo} object.
|
|
156
156
|
*
|
|
157
157
|
* To add multiple requests to the queue by extracting links from a webpage,
|
|
158
|
-
* see the {@
|
|
158
|
+
* see the {@apilink enqueueLinks} helper function.
|
|
159
159
|
*
|
|
160
|
-
* @param requestLike {@
|
|
160
|
+
* @param requestLike {@apilink Request} object or vanilla object with request data.
|
|
161
161
|
* Note that the function sets the `uniqueKey` and `id` fields to the passed Request.
|
|
162
162
|
* @param [options] Request queue operation options.
|
|
163
163
|
*/
|
|
@@ -167,9 +167,9 @@ export declare class RequestQueue {
|
|
|
167
167
|
*
|
|
168
168
|
* If a request that is passed in is already present due to its `uniqueKey` property being the same,
|
|
169
169
|
* it will not be updated. You can find out whether this happened by finding the request in the resulting
|
|
170
|
-
* {@
|
|
170
|
+
* {@apilink BatchAddRequestsResult} object.
|
|
171
171
|
*
|
|
172
|
-
* @param requestsLike {@
|
|
172
|
+
* @param requestsLike {@apilink Request} objects or vanilla objects with request data.
|
|
173
173
|
* Note that the function sets the `uniqueKey` and `id` fields to the passed requests if missing.
|
|
174
174
|
* @param [options] Request queue operation options.
|
|
175
175
|
*/
|
|
@@ -185,15 +185,15 @@ export declare class RequestQueue {
|
|
|
185
185
|
* Returns a next request in the queue to be processed, or `null` if there are no more pending requests.
|
|
186
186
|
*
|
|
187
187
|
* Once you successfully finish processing of the request, you need to call
|
|
188
|
-
* {@
|
|
188
|
+
* {@apilink RequestQueue.markRequestHandled}
|
|
189
189
|
* to mark the request as handled in the queue. If there was some error in processing the request,
|
|
190
|
-
* call {@
|
|
190
|
+
* call {@apilink RequestQueue.reclaimRequest} instead,
|
|
191
191
|
* so that the queue will give the request to some other consumer in another call to the `fetchNextRequest` function.
|
|
192
192
|
*
|
|
193
193
|
* Note that the `null` return value doesn't mean the queue processing finished,
|
|
194
194
|
* it means there are currently no pending requests.
|
|
195
195
|
* To check whether all requests in queue were finished,
|
|
196
|
-
* use {@
|
|
196
|
+
* use {@apilink RequestQueue.isFinished} instead.
|
|
197
197
|
*
|
|
198
198
|
* @returns
|
|
199
199
|
* Returns the request object or `null` if there are no more pending requests.
|
|
@@ -201,23 +201,23 @@ export declare class RequestQueue {
|
|
|
201
201
|
fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
|
|
202
202
|
/**
|
|
203
203
|
* Marks a request that was previously returned by the
|
|
204
|
-
* {@
|
|
204
|
+
* {@apilink RequestQueue.fetchNextRequest}
|
|
205
205
|
* function as handled after successful processing.
|
|
206
206
|
* Handled requests will never again be returned by the `fetchNextRequest` function.
|
|
207
207
|
*/
|
|
208
208
|
markRequestHandled(request: Request): Promise<QueueOperationInfo | null>;
|
|
209
209
|
/**
|
|
210
210
|
* Reclaims a failed request back to the queue, so that it can be returned for processing later again
|
|
211
|
-
* by another call to {@
|
|
211
|
+
* by another call to {@apilink RequestQueue.fetchNextRequest}.
|
|
212
212
|
* The request record in the queue is updated using the provided `request` parameter.
|
|
213
213
|
* For example, this lets you store the number of retries or error messages for the request.
|
|
214
214
|
*/
|
|
215
215
|
reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<QueueOperationInfo | null>;
|
|
216
216
|
/**
|
|
217
|
-
* Resolves to `true` if the next call to {@
|
|
217
|
+
* Resolves to `true` if the next call to {@apilink RequestQueue.fetchNextRequest}
|
|
218
218
|
* would return `null`, otherwise it resolves to `false`.
|
|
219
219
|
* Note that even if the queue is empty, there might be some pending requests currently being processed.
|
|
220
|
-
* If you need to ensure that there is no activity in the queue, use {@
|
|
220
|
+
* If you need to ensure that there is no activity in the queue, use {@apilink RequestQueue.isFinished}.
|
|
221
221
|
*/
|
|
222
222
|
isEmpty(): Promise<boolean>;
|
|
223
223
|
/**
|
|
@@ -290,14 +290,14 @@ export declare class RequestQueue {
|
|
|
290
290
|
getInfo(): Promise<RequestQueueInfo | undefined>;
|
|
291
291
|
/**
|
|
292
292
|
* Opens a request queue and returns a promise resolving to an instance
|
|
293
|
-
* of the {@
|
|
293
|
+
* of the {@apilink RequestQueue} class.
|
|
294
294
|
*
|
|
295
|
-
* {@
|
|
295
|
+
* {@apilink RequestQueue} represents a queue of URLs to crawl, which is stored either on local filesystem or in the cloud.
|
|
296
296
|
* The queue is used for deep crawling of websites, where you start with several URLs and then
|
|
297
297
|
* recursively follow links to other pages. The data structure supports both breadth-first
|
|
298
298
|
* and depth-first crawling orders.
|
|
299
299
|
*
|
|
300
|
-
* For more details and code examples, see the {@
|
|
300
|
+
* For more details and code examples, see the {@apilink RequestQueue} class.
|
|
301
301
|
*
|
|
302
302
|
* @param [queueIdOrName]
|
|
303
303
|
* ID or name of the request queue to be opened. If `null` or `undefined`,
|
|
@@ -67,17 +67,17 @@ exports.getRequestId = getRequestId;
|
|
|
67
67
|
* where you start with several URLs and then recursively
|
|
68
68
|
* follow links to other pages. The data structure supports both breadth-first and depth-first crawling orders.
|
|
69
69
|
*
|
|
70
|
-
* Each URL is represented using an instance of the {@
|
|
71
|
-
* The queue can only contain unique URLs. More precisely, it can only contain {@
|
|
70
|
+
* Each URL is represented using an instance of the {@apilink Request} class.
|
|
71
|
+
* The queue can only contain unique URLs. More precisely, it can only contain {@apilink Request} instances
|
|
72
72
|
* with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
|
|
73
73
|
* To add a single URL multiple times to the queue,
|
|
74
|
-
* corresponding {@
|
|
74
|
+
* corresponding {@apilink Request} objects will need to have different `uniqueKey` properties.
|
|
75
75
|
*
|
|
76
|
-
* Do not instantiate this class directly, use the {@
|
|
76
|
+
* Do not instantiate this class directly, use the {@apilink RequestQueue.open} function instead.
|
|
77
77
|
*
|
|
78
|
-
* `RequestQueue` is used by {@
|
|
79
|
-
* and {@
|
|
80
|
-
* Unlike {@
|
|
78
|
+
* `RequestQueue` is used by {@apilink BasicCrawler}, {@apilink CheerioCrawler}, {@apilink PuppeteerCrawler}
|
|
79
|
+
* and {@apilink PlaywrightCrawler} as a source of URLs to crawl.
|
|
80
|
+
* Unlike {@apilink RequestList}, `RequestQueue` supports dynamic adding and removing of requests.
|
|
81
81
|
* On the other hand, the queue is not optimized for operations that add or remove a large number of URLs in a batch.
|
|
82
82
|
*
|
|
83
83
|
* `RequestQueue` stores its data either on local disk or in the Apify Cloud,
|
|
@@ -89,7 +89,7 @@ exports.getRequestId = getRequestId;
|
|
|
89
89
|
* If the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the
|
|
90
90
|
* [Apify Request Queue](https://docs.apify.com/storage/request-queue)
|
|
91
91
|
* cloud storage. Note that you can force usage of the cloud storage also by passing the `forceCloud`
|
|
92
|
-
* option to {@
|
|
92
|
+
* option to {@apilink RequestQueue.open} function,
|
|
93
93
|
* even if the `APIFY_LOCAL_STORAGE_DIR` variable is set.
|
|
94
94
|
*
|
|
95
95
|
* **Example usage:**
|
|
@@ -244,12 +244,12 @@ class RequestQueue {
|
|
|
244
244
|
*
|
|
245
245
|
* If a request with the same `uniqueKey` property is already present in the queue,
|
|
246
246
|
* it will not be updated. You can find out whether this happened from the resulting
|
|
247
|
-
* {@
|
|
247
|
+
* {@apilink QueueOperationInfo} object.
|
|
248
248
|
*
|
|
249
249
|
* To add multiple requests to the queue by extracting links from a webpage,
|
|
250
|
-
* see the {@
|
|
250
|
+
* see the {@apilink enqueueLinks} helper function.
|
|
251
251
|
*
|
|
252
|
-
* @param requestLike {@
|
|
252
|
+
* @param requestLike {@apilink Request} object or vanilla object with request data.
|
|
253
253
|
* Note that the function sets the `uniqueKey` and `id` fields to the passed Request.
|
|
254
254
|
* @param [options] Request queue operation options.
|
|
255
255
|
*/
|
|
@@ -295,9 +295,9 @@ class RequestQueue {
|
|
|
295
295
|
*
|
|
296
296
|
* If a request that is passed in is already present due to its `uniqueKey` property being the same,
|
|
297
297
|
* it will not be updated. You can find out whether this happened by finding the request in the resulting
|
|
298
|
-
* {@
|
|
298
|
+
* {@apilink BatchAddRequestsResult} object.
|
|
299
299
|
*
|
|
300
|
-
* @param requestsLike {@
|
|
300
|
+
* @param requestsLike {@apilink Request} objects or vanilla objects with request data.
|
|
301
301
|
* Note that the function sets the `uniqueKey` and `id` fields to the passed requests if missing.
|
|
302
302
|
* @param [options] Request queue operation options.
|
|
303
303
|
*/
|
|
@@ -386,15 +386,15 @@ class RequestQueue {
|
|
|
386
386
|
* Returns a next request in the queue to be processed, or `null` if there are no more pending requests.
|
|
387
387
|
*
|
|
388
388
|
* Once you successfully finish processing of the request, you need to call
|
|
389
|
-
* {@
|
|
389
|
+
* {@apilink RequestQueue.markRequestHandled}
|
|
390
390
|
* to mark the request as handled in the queue. If there was some error in processing the request,
|
|
391
|
-
* call {@
|
|
391
|
+
* call {@apilink RequestQueue.reclaimRequest} instead,
|
|
392
392
|
* so that the queue will give the request to some other consumer in another call to the `fetchNextRequest` function.
|
|
393
393
|
*
|
|
394
394
|
* Note that the `null` return value doesn't mean the queue processing finished,
|
|
395
395
|
* it means there are currently no pending requests.
|
|
396
396
|
* To check whether all requests in queue were finished,
|
|
397
|
-
* use {@
|
|
397
|
+
* use {@apilink RequestQueue.isFinished} instead.
|
|
398
398
|
*
|
|
399
399
|
* @returns
|
|
400
400
|
* Returns the request object or `null` if there are no more pending requests.
|
|
@@ -451,7 +451,7 @@ class RequestQueue {
|
|
|
451
451
|
}
|
|
452
452
|
/**
|
|
453
453
|
* Marks a request that was previously returned by the
|
|
454
|
-
* {@
|
|
454
|
+
* {@apilink RequestQueue.fetchNextRequest}
|
|
455
455
|
* function as handled after successful processing.
|
|
456
456
|
* Handled requests will never again be returned by the `fetchNextRequest` function.
|
|
457
457
|
*/
|
|
@@ -480,7 +480,7 @@ class RequestQueue {
|
|
|
480
480
|
}
|
|
481
481
|
/**
|
|
482
482
|
* Reclaims a failed request back to the queue, so that it can be returned for processing later again
|
|
483
|
-
* by another call to {@
|
|
483
|
+
* by another call to {@apilink RequestQueue.fetchNextRequest}.
|
|
484
484
|
* The request record in the queue is updated using the provided `request` parameter.
|
|
485
485
|
* For example, this lets you store the number of retries or error messages for the request.
|
|
486
486
|
*/
|
|
@@ -517,10 +517,10 @@ class RequestQueue {
|
|
|
517
517
|
return queueOperationInfo;
|
|
518
518
|
}
|
|
519
519
|
/**
|
|
520
|
-
* Resolves to `true` if the next call to {@
|
|
520
|
+
* Resolves to `true` if the next call to {@apilink RequestQueue.fetchNextRequest}
|
|
521
521
|
* would return `null`, otherwise it resolves to `false`.
|
|
522
522
|
* Note that even if the queue is empty, there might be some pending requests currently being processed.
|
|
523
|
-
* If you need to ensure that there is no activity in the queue, use {@
|
|
523
|
+
* If you need to ensure that there is no activity in the queue, use {@apilink RequestQueue.isFinished}.
|
|
524
524
|
*/
|
|
525
525
|
async isEmpty() {
|
|
526
526
|
await this._ensureHeadIsNonEmpty();
|
|
@@ -709,14 +709,14 @@ class RequestQueue {
|
|
|
709
709
|
}
|
|
710
710
|
/**
|
|
711
711
|
* Opens a request queue and returns a promise resolving to an instance
|
|
712
|
-
* of the {@
|
|
712
|
+
* of the {@apilink RequestQueue} class.
|
|
713
713
|
*
|
|
714
|
-
* {@
|
|
714
|
+
* {@apilink RequestQueue} represents a queue of URLs to crawl, which is stored either on local filesystem or in the cloud.
|
|
715
715
|
* The queue is used for deep crawling of websites, where you start with several URLs and then
|
|
716
716
|
* recursively follow links to other pages. The data structure supports both breadth-first
|
|
717
717
|
* and depth-first crawling orders.
|
|
718
718
|
*
|
|
719
|
-
* For more details and code examples, see the {@
|
|
719
|
+
* For more details and code examples, see the {@apilink RequestQueue} class.
|
|
720
720
|
*
|
|
721
721
|
* @param [queueIdOrName]
|
|
722
722
|
* ID or name of the request queue to be opened. If `null` or `undefined`,
|
package/storages/utils.d.ts
CHANGED
|
@@ -4,7 +4,7 @@ import { Configuration } from '../configuration';
|
|
|
4
4
|
* Purging will remove all the files in all storages except for INPUT.json in the default KV store.
|
|
5
5
|
*
|
|
6
6
|
* Purging of storages is happening automatically when we run our crawler (or when we open some storage
|
|
7
|
-
* explicitly, e.g. via `RequestList.open()`). We can disable that via `purgeOnStart` {@
|
|
7
|
+
* explicitly, e.g. via `RequestList.open()`). We can disable that via `purgeOnStart` {@apilink Configuration}
|
|
8
8
|
* option or by setting `CRAWLEE_PURGE_ON_START` environment variable to `0` or `false`.
|
|
9
9
|
*
|
|
10
10
|
* This is a shortcut for running (optional) `purge` method on the StorageClient interface, in other words
|
package/storages/utils.js
CHANGED
|
@@ -7,7 +7,7 @@ const configuration_1 = require("../configuration");
|
|
|
7
7
|
* Purging will remove all the files in all storages except for INPUT.json in the default KV store.
|
|
8
8
|
*
|
|
9
9
|
* Purging of storages is happening automatically when we run our crawler (or when we open some storage
|
|
10
|
-
* explicitly, e.g. via `RequestList.open()`). We can disable that via `purgeOnStart` {@
|
|
10
|
+
* explicitly, e.g. via `RequestList.open()`). We can disable that via `purgeOnStart` {@apilink Configuration}
|
|
11
11
|
* option or by setting `CRAWLEE_PURGE_ON_START` environment variable to `0` or `false`.
|
|
12
12
|
*
|
|
13
13
|
* This is a shortcut for running (optional) `purge` method on the StorageClient interface, in other words
|