@crawlee/core 3.0.3-beta.10 → 3.0.3-beta.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/autoscaled_pool.d.ts +16 -16
- package/autoscaling/autoscaled_pool.js +13 -13
- package/autoscaling/snapshotter.d.ts +1 -1
- package/autoscaling/snapshotter.js +1 -1
- package/autoscaling/system_status.d.ts +12 -12
- package/autoscaling/system_status.js +11 -11
- package/configuration.d.ts +10 -10
- package/configuration.js +4 -4
- package/crawlers/crawler_commons.d.ts +2 -2
- package/crawlers/statistics.d.ts +1 -1
- package/crawlers/statistics.js +1 -1
- package/enqueue_links/enqueue_links.d.ts +14 -13
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +6 -5
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +2 -2
- package/package.json +4 -4
- package/proxy_configuration.d.ts +11 -11
- package/proxy_configuration.js +8 -8
- package/request.d.ts +3 -3
- package/request.js +2 -2
- package/session_pool/session.d.ts +1 -1
- package/session_pool/session_pool.d.ts +12 -12
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +10 -10
- package/storages/dataset.d.ts +14 -14
- package/storages/dataset.js +8 -8
- package/storages/key_value_store.d.ts +24 -24
- package/storages/key_value_store.js +20 -20
- package/storages/request_list.d.ts +33 -33
- package/storages/request_list.js +23 -23
- package/storages/request_queue.d.ts +25 -25
- package/storages/request_queue.js +23 -23
- package/storages/utils.d.ts +1 -1
- package/storages/utils.js +1 -1
- package/tsconfig.build.tsbuildinfo +1 -1
|
@@ -44,16 +44,16 @@ exports.maybeStringify = maybeStringify;
|
|
|
44
44
|
* for saving screenshots, crawler inputs and outputs, web pages, PDFs or to persist the state of crawlers.
|
|
45
45
|
*
|
|
46
46
|
* Do not instantiate this class directly, use the
|
|
47
|
-
* {@
|
|
47
|
+
* {@apilink KeyValueStore.open} function instead.
|
|
48
48
|
*
|
|
49
49
|
* Each crawler run is associated with a default key-value store, which is created exclusively
|
|
50
50
|
* for the run. By convention, the crawler input and output are stored into the
|
|
51
51
|
* default key-value store under the `INPUT` and `OUTPUT` key, respectively.
|
|
52
52
|
* Typically, input and output are JSON files, although it can be any other format.
|
|
53
53
|
* To access the default key-value store directly, you can use the
|
|
54
|
-
* {@
|
|
54
|
+
* {@apilink KeyValueStore.getValue} and {@apilink KeyValueStore.setValue} convenience functions.
|
|
55
55
|
*
|
|
56
|
-
* To access the input, you can also use the {@
|
|
56
|
+
* To access the input, you can also use the {@apilink KeyValueStore.getInput} convenience function.
|
|
57
57
|
*
|
|
58
58
|
* `KeyValueStore` stores its data on a local disk.
|
|
59
59
|
*
|
|
@@ -153,7 +153,7 @@ class KeyValueStore {
|
|
|
153
153
|
* If the record does not exist, the function resolves to `null`.
|
|
154
154
|
*
|
|
155
155
|
* To save or delete a value in the key-value store, use the
|
|
156
|
-
* {@
|
|
156
|
+
* {@apilink KeyValueStore.setValue} function.
|
|
157
157
|
*
|
|
158
158
|
* **Example usage:**
|
|
159
159
|
*
|
|
@@ -223,7 +223,7 @@ class KeyValueStore {
|
|
|
223
223
|
* regardless whether the record existed or not.
|
|
224
224
|
*
|
|
225
225
|
* To retrieve a value from the key-value store, use the
|
|
226
|
-
* {@
|
|
226
|
+
* {@apilink KeyValueStore.getValue} function.
|
|
227
227
|
*
|
|
228
228
|
* **IMPORTANT:** Always make sure to use the `await` keyword when calling `setValue()`,
|
|
229
229
|
* otherwise the crawler process might finish before the value is stored!
|
|
@@ -333,13 +333,13 @@ class KeyValueStore {
|
|
|
333
333
|
: undefined; // [].forEach() returns undefined.
|
|
334
334
|
}
|
|
335
335
|
/**
|
|
336
|
-
* Opens a key-value store and returns a promise resolving to an instance of the {@
|
|
336
|
+
* Opens a key-value store and returns a promise resolving to an instance of the {@apilink KeyValueStore} class.
|
|
337
337
|
*
|
|
338
338
|
* Key-value stores are used to store records or files, along with their MIME content type.
|
|
339
339
|
* The records are stored and retrieved using a unique key.
|
|
340
340
|
* The actual data is stored either on a local filesystem or in the Apify cloud.
|
|
341
341
|
*
|
|
342
|
-
* For more details and code examples, see the {@
|
|
342
|
+
* For more details and code examples, see the {@apilink KeyValueStore} class.
|
|
343
343
|
*
|
|
344
344
|
* @param [storeIdOrName]
|
|
345
345
|
* ID or name of the key-value store to be opened. If `null` or `undefined`,
|
|
@@ -356,9 +356,9 @@ class KeyValueStore {
|
|
|
356
356
|
return manager.openStorage(storeIdOrName);
|
|
357
357
|
}
|
|
358
358
|
/**
|
|
359
|
-
* Gets a value from the default {@
|
|
359
|
+
* Gets a value from the default {@apilink KeyValueStore} associated with the current crawler run.
|
|
360
360
|
*
|
|
361
|
-
* This is just a convenient shortcut for {@
|
|
361
|
+
* This is just a convenient shortcut for {@apilink KeyValueStore.getValue}.
|
|
362
362
|
* For example, calling the following code:
|
|
363
363
|
* ```javascript
|
|
364
364
|
* const value = await KeyValueStore.getValue('my-key');
|
|
@@ -370,10 +370,10 @@ class KeyValueStore {
|
|
|
370
370
|
* const value = await store.getValue('my-key');
|
|
371
371
|
* ```
|
|
372
372
|
*
|
|
373
|
-
* To store the value to the default key-value store, you can use the {@
|
|
373
|
+
* To store the value to the default key-value store, you can use the {@apilink KeyValueStore.setValue} function.
|
|
374
374
|
*
|
|
375
|
-
* For more information, see {@
|
|
376
|
-
* and {@
|
|
375
|
+
* For more information, see {@apilink KeyValueStore.open}
|
|
376
|
+
* and {@apilink KeyValueStore.getValue}.
|
|
377
377
|
*
|
|
378
378
|
* @param key Unique record key.
|
|
379
379
|
* @param defaultValue Fallback that will be returned if no value if present in the storage.
|
|
@@ -393,9 +393,9 @@ class KeyValueStore {
|
|
|
393
393
|
return store.getAutoSavedValue(key, defaultValue);
|
|
394
394
|
}
|
|
395
395
|
/**
|
|
396
|
-
* Stores or deletes a value in the default {@
|
|
396
|
+
* Stores or deletes a value in the default {@apilink KeyValueStore} associated with the current crawler run.
|
|
397
397
|
*
|
|
398
|
-
* This is just a convenient shortcut for {@
|
|
398
|
+
* This is just a convenient shortcut for {@apilink KeyValueStore.setValue}.
|
|
399
399
|
* For example, calling the following code:
|
|
400
400
|
* ```javascript
|
|
401
401
|
* await KeyValueStore.setValue('OUTPUT', { foo: "bar" });
|
|
@@ -407,10 +407,10 @@ class KeyValueStore {
|
|
|
407
407
|
* await store.setValue('OUTPUT', { foo: "bar" });
|
|
408
408
|
* ```
|
|
409
409
|
*
|
|
410
|
-
* To get a value from the default key-value store, you can use the {@
|
|
410
|
+
* To get a value from the default key-value store, you can use the {@apilink KeyValueStore.getValue} function.
|
|
411
411
|
*
|
|
412
|
-
* For more information, see {@
|
|
413
|
-
* and {@
|
|
412
|
+
* For more information, see {@apilink KeyValueStore.open}
|
|
413
|
+
* and {@apilink KeyValueStore.getValue}.
|
|
414
414
|
*
|
|
415
415
|
* @param key
|
|
416
416
|
* Unique record key.
|
|
@@ -428,7 +428,7 @@ class KeyValueStore {
|
|
|
428
428
|
return store.setValue(key, value, options);
|
|
429
429
|
}
|
|
430
430
|
/**
|
|
431
|
-
* Gets the crawler input value from the default {@
|
|
431
|
+
* Gets the crawler input value from the default {@apilink KeyValueStore} associated with the current crawler run.
|
|
432
432
|
*
|
|
433
433
|
* This is just a convenient shortcut for [`keyValueStore.getValue('INPUT')`](core/class/KeyValueStore#getValue).
|
|
434
434
|
* For example, calling the following code:
|
|
@@ -446,8 +446,8 @@ class KeyValueStore {
|
|
|
446
446
|
* If you need to use the input multiple times in your crawler,
|
|
447
447
|
* it is far more efficient to read it once and store it locally.
|
|
448
448
|
*
|
|
449
|
-
* For more information, see {@
|
|
450
|
-
* and {@
|
|
449
|
+
* For more information, see {@apilink KeyValueStore.open}
|
|
450
|
+
* and {@apilink KeyValueStore.getValue}.
|
|
451
451
|
*
|
|
452
452
|
* @returns
|
|
453
453
|
* Returns a promise that resolves to an object, string
|
|
@@ -9,8 +9,8 @@ export declare const STATE_PERSISTENCE_KEY = "REQUEST_LIST_STATE";
|
|
|
9
9
|
export declare const REQUESTS_PERSISTENCE_KEY = "REQUEST_LIST_REQUESTS";
|
|
10
10
|
export interface RequestListOptions {
|
|
11
11
|
/**
|
|
12
|
-
* An array of sources of URLs for the {@
|
|
13
|
-
* plain objects that define at least the `url` property, or an array of {@
|
|
12
|
+
* An array of sources of URLs for the {@apilink RequestList}. It can be either an array of strings,
|
|
13
|
+
* plain objects that define at least the `url` property, or an array of {@apilink Request} instances.
|
|
14
14
|
*
|
|
15
15
|
* **IMPORTANT:** The `sources` array will be consumed (left empty) after `RequestList` initializes.
|
|
16
16
|
* This is a measure to prevent memory leaks in situations when millions of sources are
|
|
@@ -44,15 +44,15 @@ export interface RequestListOptions {
|
|
|
44
44
|
sources?: Source[];
|
|
45
45
|
/**
|
|
46
46
|
* A function that will be called to get the sources for the `RequestList`, but only if `RequestList`
|
|
47
|
-
* was not able to fetch their persisted version (see {@
|
|
48
|
-
* It must return an `Array` of {@
|
|
47
|
+
* was not able to fetch their persisted version (see {@apilink RequestListOptions.persistRequestsKey}).
|
|
48
|
+
* It must return an `Array` of {@apilink Request} or {@apilink RequestOptions}.
|
|
49
49
|
*
|
|
50
50
|
* This is very useful in a scenario when getting the sources is a resource intensive or time consuming
|
|
51
51
|
* task, such as fetching URLs from multiple sitemaps or parsing URLs from large datasets. Using the
|
|
52
52
|
* `sourcesFunction` in combination with `persistStateKey` and `persistRequestsKey` will allow you to
|
|
53
53
|
* fetch and parse those URLs only once, saving valuable time when your crawler migrates or restarts.
|
|
54
54
|
*
|
|
55
|
-
* If both {@
|
|
55
|
+
* If both {@apilink RequestListOptions.sources} and {@apilink RequestListOptions.sourcesFunction} are provided,
|
|
56
56
|
* the sources returned by the function will be added after the `sources`.
|
|
57
57
|
*
|
|
58
58
|
* **Example:**
|
|
@@ -99,12 +99,12 @@ export interface RequestListOptions {
|
|
|
99
99
|
persistStateKey?: string;
|
|
100
100
|
/**
|
|
101
101
|
* Identifies the key in the default key-value store under which the `RequestList` persists its
|
|
102
|
-
* Requests during the {@
|
|
102
|
+
* Requests during the {@apilink RequestList.initialize} call.
|
|
103
103
|
* This is necessary if `persistStateKey` is set and the source URLs might potentially change,
|
|
104
104
|
* to ensure consistency of the source URLs and state object. However, it comes with some
|
|
105
105
|
* storage and performance overheads.
|
|
106
106
|
*
|
|
107
|
-
* If `persistRequestsKey` is not set, {@
|
|
107
|
+
* If `persistRequestsKey` is not set, {@apilink RequestList.initialize} will always fetch the sources
|
|
108
108
|
* from their origin, check that they are consistent with the restored state (if any)
|
|
109
109
|
* and throw an error if they are not.
|
|
110
110
|
*/
|
|
@@ -130,7 +130,7 @@ export interface RequestListOptions {
|
|
|
130
130
|
state?: RequestListState;
|
|
131
131
|
/**
|
|
132
132
|
* By default, `RequestList` will deduplicate the provided URLs. Default deduplication is based
|
|
133
|
-
* on the `uniqueKey` property of passed source {@
|
|
133
|
+
* on the `uniqueKey` property of passed source {@apilink Request} objects.
|
|
134
134
|
*
|
|
135
135
|
* If the property is not present, it is generated by normalizing the URL. If present, it is kept intact.
|
|
136
136
|
* In any case, only one request per `uniqueKey` is added to the `RequestList` resulting in removal
|
|
@@ -150,19 +150,19 @@ export interface RequestListOptions {
|
|
|
150
150
|
/**
|
|
151
151
|
* Represents a static list of URLs to crawl.
|
|
152
152
|
* The URLs can be provided either in code or parsed from a text file hosted on the web.
|
|
153
|
-
* `RequestList` is used by {@
|
|
154
|
-
* and {@
|
|
153
|
+
* `RequestList` is used by {@apilink BasicCrawler}, {@apilink CheerioCrawler}, {@apilink PuppeteerCrawler}
|
|
154
|
+
* and {@apilink PlaywrightCrawler} as a source of URLs to crawl.
|
|
155
155
|
*
|
|
156
|
-
* Each URL is represented using an instance of the {@
|
|
156
|
+
* Each URL is represented using an instance of the {@apilink Request} class.
|
|
157
157
|
* The list can only contain unique URLs. More precisely, it can only contain `Request` instances
|
|
158
158
|
* with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
|
|
159
|
-
* To add a single URL to the list multiple times, corresponding {@
|
|
159
|
+
* To add a single URL to the list multiple times, corresponding {@apilink Request} objects will need to have different
|
|
160
160
|
* `uniqueKey` properties. You can use the `keepDuplicateUrls` option to do this for you when initializing the
|
|
161
161
|
* `RequestList` from sources.
|
|
162
162
|
*
|
|
163
|
-
* Once you create an instance of `RequestList`, you need to call the {@
|
|
163
|
+
* Once you create an instance of `RequestList`, you need to call the {@apilink RequestList.initialize} function
|
|
164
164
|
* before the instance can be used. After that, no more URLs can be added to the list.
|
|
165
|
-
* Unlike {@
|
|
165
|
+
* Unlike {@apilink RequestQueue}, `RequestList` is static but it can contain even millions of URLs.
|
|
166
166
|
* > Note that `RequestList` can be used together with `RequestQueue` by the same crawler.
|
|
167
167
|
* > In such cases, each request from `RequestList` is enqueued into `RequestQueue` first and then consumed from the latter.
|
|
168
168
|
* > This is necessary to avoid the same URL being processed more than once (from the list first and then possibly from the queue).
|
|
@@ -171,9 +171,9 @@ export interface RequestListOptions {
|
|
|
171
171
|
*
|
|
172
172
|
* `RequestList` has an internal state where it stores information about which requests were already handled,
|
|
173
173
|
* which are in progress and which were reclaimed. The state may be automatically persisted to the default
|
|
174
|
-
* {@
|
|
174
|
+
* {@apilink KeyValueStore} by setting the `persistStateKey` option so that if the Node.js process is restarted,
|
|
175
175
|
* the crawling can continue where it left off. The automated persisting is launched upon receiving the `persistState`
|
|
176
|
-
* event that is periodically emitted by {@
|
|
176
|
+
* event that is periodically emitted by {@apilink EventManager}.
|
|
177
177
|
*
|
|
178
178
|
* The internal state is closely tied to the provided sources (URLs). If the sources change on crawler restart, the state will become corrupted and
|
|
179
179
|
* `RequestList` will raise an exception. This typically happens when the sources is a list of URLs downloaded from the web.
|
|
@@ -279,7 +279,7 @@ export declare class RequestList {
|
|
|
279
279
|
*/
|
|
280
280
|
protected _addRequestsFromSources(): Promise<void>;
|
|
281
281
|
/**
|
|
282
|
-
* Persists the current state of the `RequestList` into the default {@
|
|
282
|
+
* Persists the current state of the `RequestList` into the default {@apilink KeyValueStore}.
|
|
283
283
|
* The state is persisted automatically in regular intervals, but calling this method manually
|
|
284
284
|
* is useful in cases where you want to have the most current state available after you pause
|
|
285
285
|
* or stop fetching its requests. For example after you pause or abort a crawl. Or just before
|
|
@@ -307,7 +307,7 @@ export declare class RequestList {
|
|
|
307
307
|
*/
|
|
308
308
|
getState(): RequestListState;
|
|
309
309
|
/**
|
|
310
|
-
* Resolves to `true` if the next call to {@
|
|
310
|
+
* Resolves to `true` if the next call to {@apilink RequestList.fetchNextRequest} function
|
|
311
311
|
* would return `null`, otherwise it resolves to `false`.
|
|
312
312
|
* Note that even if the list is empty, there might be some pending requests currently being processed.
|
|
313
313
|
*/
|
|
@@ -317,8 +317,8 @@ export declare class RequestList {
|
|
|
317
317
|
*/
|
|
318
318
|
isFinished(): Promise<boolean>;
|
|
319
319
|
/**
|
|
320
|
-
* Gets the next {@
|
|
321
|
-
* using the {@
|
|
320
|
+
* Gets the next {@apilink Request} to process. First, the function gets a request previously reclaimed
|
|
321
|
+
* using the {@apilink RequestList.reclaimRequest} function, if there is any.
|
|
322
322
|
* Otherwise it gets the next request from sources.
|
|
323
323
|
*
|
|
324
324
|
* The function's `Promise` resolves to `null` if there are no more
|
|
@@ -372,14 +372,14 @@ export declare class RequestList {
|
|
|
372
372
|
handledCount(): number;
|
|
373
373
|
/**
|
|
374
374
|
* Opens a request list and returns a promise resolving to an instance
|
|
375
|
-
* of the {@
|
|
375
|
+
* of the {@apilink RequestList} class that is already initialized.
|
|
376
376
|
*
|
|
377
|
-
* {@
|
|
377
|
+
* {@apilink RequestList} represents a list of URLs to crawl, which is always stored in memory.
|
|
378
378
|
* To enable picking up where left off after a process restart, the request list sources
|
|
379
379
|
* are persisted to the key-value store at initialization of the list. Then, while crawling,
|
|
380
380
|
* a small state object is regularly persisted to keep track of the crawling status.
|
|
381
381
|
*
|
|
382
|
-
* For more details and code examples, see the {@
|
|
382
|
+
* For more details and code examples, see the {@apilink RequestList} class.
|
|
383
383
|
*
|
|
384
384
|
* **Example usage:**
|
|
385
385
|
*
|
|
@@ -405,23 +405,23 @@ export declare class RequestList {
|
|
|
405
405
|
* If `null`, the list will not be persisted and will only be stored in memory. Process restart
|
|
406
406
|
* will then cause the list to be crawled again from the beginning. We suggest always using a name.
|
|
407
407
|
* @param [sources]
|
|
408
|
-
* An array of sources of URLs for the {@
|
|
409
|
-
* plain objects that define at least the `url` property, or an array of {@
|
|
408
|
+
* An array of sources of URLs for the {@apilink RequestList}. It can be either an array of strings,
|
|
409
|
+
* plain objects that define at least the `url` property, or an array of {@apilink Request} instances.
|
|
410
410
|
*
|
|
411
|
-
* **IMPORTANT:** The `sources` array will be consumed (left empty) after {@
|
|
411
|
+
* **IMPORTANT:** The `sources` array will be consumed (left empty) after {@apilink RequestList} initializes.
|
|
412
412
|
* This is a measure to prevent memory leaks in situations when millions of sources are
|
|
413
413
|
* added.
|
|
414
414
|
*
|
|
415
415
|
* Additionally, the `requestsFromUrl` property may be used instead of `url`,
|
|
416
|
-
* which will instruct {@
|
|
416
|
+
* which will instruct {@apilink RequestList} to download the source URLs from a given remote location.
|
|
417
417
|
* The URLs will be parsed from the received response. In this case you can limit the URLs
|
|
418
418
|
* using `regex` parameter containing regular expression pattern for URLs to be included.
|
|
419
419
|
*
|
|
420
|
-
* For details, see the {@
|
|
420
|
+
* For details, see the {@apilink RequestListOptions.sources}
|
|
421
421
|
* @param [options]
|
|
422
|
-
* The {@
|
|
423
|
-
* the {@
|
|
424
|
-
* options and the `sources` parameter supersedes the {@
|
|
422
|
+
* The {@apilink RequestList} options. Note that the `listName` parameter supersedes
|
|
423
|
+
* the {@apilink RequestListOptions.persistStateKey} and {@apilink RequestListOptions.persistRequestsKey}
|
|
424
|
+
* options and the `sources` parameter supersedes the {@apilink RequestListOptions.sources} option.
|
|
425
425
|
*/
|
|
426
426
|
static open(listNameOrOptions: string | null | RequestListOptions, sources?: Source[], options?: RequestListOptions): Promise<RequestList>;
|
|
427
427
|
/**
|
|
@@ -430,8 +430,8 @@ export declare class RequestList {
|
|
|
430
430
|
private _downloadListOfUrls;
|
|
431
431
|
}
|
|
432
432
|
/**
|
|
433
|
-
* Represents state of a {@
|
|
434
|
-
* You can obtain the state by calling {@
|
|
433
|
+
* Represents state of a {@apilink RequestList}. It can be used to resume a {@apilink RequestList} which has been previously processed.
|
|
434
|
+
* You can obtain the state by calling {@apilink RequestList.getState} and receive an object with
|
|
435
435
|
* the following structure:
|
|
436
436
|
*
|
|
437
437
|
* ```
|
package/storages/request_list.js
CHANGED
|
@@ -18,19 +18,19 @@ const CONTENT_TYPE_BINARY = 'application/octet-stream';
|
|
|
18
18
|
/**
|
|
19
19
|
* Represents a static list of URLs to crawl.
|
|
20
20
|
* The URLs can be provided either in code or parsed from a text file hosted on the web.
|
|
21
|
-
* `RequestList` is used by {@
|
|
22
|
-
* and {@
|
|
21
|
+
* `RequestList` is used by {@apilink BasicCrawler}, {@apilink CheerioCrawler}, {@apilink PuppeteerCrawler}
|
|
22
|
+
* and {@apilink PlaywrightCrawler} as a source of URLs to crawl.
|
|
23
23
|
*
|
|
24
|
-
* Each URL is represented using an instance of the {@
|
|
24
|
+
* Each URL is represented using an instance of the {@apilink Request} class.
|
|
25
25
|
* The list can only contain unique URLs. More precisely, it can only contain `Request` instances
|
|
26
26
|
* with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
|
|
27
|
-
* To add a single URL to the list multiple times, corresponding {@
|
|
27
|
+
* To add a single URL to the list multiple times, corresponding {@apilink Request} objects will need to have different
|
|
28
28
|
* `uniqueKey` properties. You can use the `keepDuplicateUrls` option to do this for you when initializing the
|
|
29
29
|
* `RequestList` from sources.
|
|
30
30
|
*
|
|
31
|
-
* Once you create an instance of `RequestList`, you need to call the {@
|
|
31
|
+
* Once you create an instance of `RequestList`, you need to call the {@apilink RequestList.initialize} function
|
|
32
32
|
* before the instance can be used. After that, no more URLs can be added to the list.
|
|
33
|
-
* Unlike {@
|
|
33
|
+
* Unlike {@apilink RequestQueue}, `RequestList` is static but it can contain even millions of URLs.
|
|
34
34
|
* > Note that `RequestList` can be used together with `RequestQueue` by the same crawler.
|
|
35
35
|
* > In such cases, each request from `RequestList` is enqueued into `RequestQueue` first and then consumed from the latter.
|
|
36
36
|
* > This is necessary to avoid the same URL being processed more than once (from the list first and then possibly from the queue).
|
|
@@ -39,9 +39,9 @@ const CONTENT_TYPE_BINARY = 'application/octet-stream';
|
|
|
39
39
|
*
|
|
40
40
|
* `RequestList` has an internal state where it stores information about which requests were already handled,
|
|
41
41
|
* which are in progress and which were reclaimed. The state may be automatically persisted to the default
|
|
42
|
-
* {@
|
|
42
|
+
* {@apilink KeyValueStore} by setting the `persistStateKey` option so that if the Node.js process is restarted,
|
|
43
43
|
* the crawling can continue where it left off. The automated persisting is launched upon receiving the `persistState`
|
|
44
|
-
* event that is periodically emitted by {@
|
|
44
|
+
* event that is periodically emitted by {@apilink EventManager}.
|
|
45
45
|
*
|
|
46
46
|
* The internal state is closely tied to the provided sources (URLs). If the sources change on crawler restart, the state will become corrupted and
|
|
47
47
|
* `RequestList` will raise an exception. This typically happens when the sources is a list of URLs downloaded from the web.
|
|
@@ -338,7 +338,7 @@ class RequestList {
|
|
|
338
338
|
}
|
|
339
339
|
}
|
|
340
340
|
/**
|
|
341
|
-
* Persists the current state of the `RequestList` into the default {@
|
|
341
|
+
* Persists the current state of the `RequestList` into the default {@apilink KeyValueStore}.
|
|
342
342
|
* The state is persisted automatically in regular intervals, but calling this method manually
|
|
343
343
|
* is useful in cases where you want to have the most current state available after you pause
|
|
344
344
|
* or stop fetching its requests. For example after you pause or abort a crawl. Or just before
|
|
@@ -463,7 +463,7 @@ class RequestList {
|
|
|
463
463
|
};
|
|
464
464
|
}
|
|
465
465
|
/**
|
|
466
|
-
* Resolves to `true` if the next call to {@
|
|
466
|
+
* Resolves to `true` if the next call to {@apilink RequestList.fetchNextRequest} function
|
|
467
467
|
* would return `null`, otherwise it resolves to `false`.
|
|
468
468
|
* Note that even if the list is empty, there might be some pending requests currently being processed.
|
|
469
469
|
*/
|
|
@@ -479,8 +479,8 @@ class RequestList {
|
|
|
479
479
|
return this.inProgress.size === 0 && this.nextIndex >= this.requests.length;
|
|
480
480
|
}
|
|
481
481
|
/**
|
|
482
|
-
* Gets the next {@
|
|
483
|
-
* using the {@
|
|
482
|
+
* Gets the next {@apilink Request} to process. First, the function gets a request previously reclaimed
|
|
483
|
+
* using the {@apilink RequestList.reclaimRequest} function, if there is any.
|
|
484
484
|
* Otherwise it gets the next request from sources.
|
|
485
485
|
*
|
|
486
486
|
* The function's `Promise` resolves to `null` if there are no more
|
|
@@ -650,14 +650,14 @@ class RequestList {
|
|
|
650
650
|
}
|
|
651
651
|
/**
|
|
652
652
|
* Opens a request list and returns a promise resolving to an instance
|
|
653
|
-
* of the {@
|
|
653
|
+
* of the {@apilink RequestList} class that is already initialized.
|
|
654
654
|
*
|
|
655
|
-
* {@
|
|
655
|
+
* {@apilink RequestList} represents a list of URLs to crawl, which is always stored in memory.
|
|
656
656
|
* To enable picking up where left off after a process restart, the request list sources
|
|
657
657
|
* are persisted to the key-value store at initialization of the list. Then, while crawling,
|
|
658
658
|
* a small state object is regularly persisted to keep track of the crawling status.
|
|
659
659
|
*
|
|
660
|
-
* For more details and code examples, see the {@
|
|
660
|
+
* For more details and code examples, see the {@apilink RequestList} class.
|
|
661
661
|
*
|
|
662
662
|
* **Example usage:**
|
|
663
663
|
*
|
|
@@ -683,23 +683,23 @@ class RequestList {
|
|
|
683
683
|
* If `null`, the list will not be persisted and will only be stored in memory. Process restart
|
|
684
684
|
* will then cause the list to be crawled again from the beginning. We suggest always using a name.
|
|
685
685
|
* @param [sources]
|
|
686
|
-
* An array of sources of URLs for the {@
|
|
687
|
-
* plain objects that define at least the `url` property, or an array of {@
|
|
686
|
+
* An array of sources of URLs for the {@apilink RequestList}. It can be either an array of strings,
|
|
687
|
+
* plain objects that define at least the `url` property, or an array of {@apilink Request} instances.
|
|
688
688
|
*
|
|
689
|
-
* **IMPORTANT:** The `sources` array will be consumed (left empty) after {@
|
|
689
|
+
* **IMPORTANT:** The `sources` array will be consumed (left empty) after {@apilink RequestList} initializes.
|
|
690
690
|
* This is a measure to prevent memory leaks in situations when millions of sources are
|
|
691
691
|
* added.
|
|
692
692
|
*
|
|
693
693
|
* Additionally, the `requestsFromUrl` property may be used instead of `url`,
|
|
694
|
-
* which will instruct {@
|
|
694
|
+
* which will instruct {@apilink RequestList} to download the source URLs from a given remote location.
|
|
695
695
|
* The URLs will be parsed from the received response. In this case you can limit the URLs
|
|
696
696
|
* using `regex` parameter containing regular expression pattern for URLs to be included.
|
|
697
697
|
*
|
|
698
|
-
* For details, see the {@
|
|
698
|
+
* For details, see the {@apilink RequestListOptions.sources}
|
|
699
699
|
* @param [options]
|
|
700
|
-
* The {@
|
|
701
|
-
* the {@
|
|
702
|
-
* options and the `sources` parameter supersedes the {@
|
|
700
|
+
* The {@apilink RequestList} options. Note that the `listName` parameter supersedes
|
|
701
|
+
* the {@apilink RequestListOptions.persistStateKey} and {@apilink RequestListOptions.persistRequestsKey}
|
|
702
|
+
* options and the `sources` parameter supersedes the {@apilink RequestListOptions.sources} option.
|
|
703
703
|
*/
|
|
704
704
|
static async open(listNameOrOptions, sources, options = {}) {
|
|
705
705
|
if (listNameOrOptions != null && typeof listNameOrOptions === 'object') {
|
|
@@ -40,7 +40,7 @@ export declare const STORAGE_CONSISTENCY_DELAY_MILLIS = 3000;
|
|
|
40
40
|
export declare function getRequestId(uniqueKey: string): string;
|
|
41
41
|
/**
|
|
42
42
|
* A helper class that is used to report results from various
|
|
43
|
-
* {@
|
|
43
|
+
* {@apilink RequestQueue} functions as well as {@apilink enqueueLinks}.
|
|
44
44
|
*/
|
|
45
45
|
export interface QueueOperationInfo {
|
|
46
46
|
/** Indicates if request was already present in the queue. */
|
|
@@ -56,7 +56,7 @@ export interface RequestQueueOperationOptions {
|
|
|
56
56
|
* If set to `true`:
|
|
57
57
|
* - while adding the request to the queue: the request will be added to the foremost position in the queue.
|
|
58
58
|
* - while reclaiming the request: the request will be placed to the beginning of the queue, so that it's returned
|
|
59
|
-
* in the next call to {@
|
|
59
|
+
* in the next call to {@apilink RequestQueue.fetchNextRequest}.
|
|
60
60
|
* By default, it's put to the end of the queue.
|
|
61
61
|
* @default false
|
|
62
62
|
*/
|
|
@@ -67,17 +67,17 @@ export interface RequestQueueOperationOptions {
|
|
|
67
67
|
* where you start with several URLs and then recursively
|
|
68
68
|
* follow links to other pages. The data structure supports both breadth-first and depth-first crawling orders.
|
|
69
69
|
*
|
|
70
|
-
* Each URL is represented using an instance of the {@
|
|
71
|
-
* The queue can only contain unique URLs. More precisely, it can only contain {@
|
|
70
|
+
* Each URL is represented using an instance of the {@apilink Request} class.
|
|
71
|
+
* The queue can only contain unique URLs. More precisely, it can only contain {@apilink Request} instances
|
|
72
72
|
* with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
|
|
73
73
|
* To add a single URL multiple times to the queue,
|
|
74
|
-
* corresponding {@
|
|
74
|
+
* corresponding {@apilink Request} objects will need to have different `uniqueKey` properties.
|
|
75
75
|
*
|
|
76
|
-
* Do not instantiate this class directly, use the {@
|
|
76
|
+
* Do not instantiate this class directly, use the {@apilink RequestQueue.open} function instead.
|
|
77
77
|
*
|
|
78
|
-
* `RequestQueue` is used by {@
|
|
79
|
-
* and {@
|
|
80
|
-
* Unlike {@
|
|
78
|
+
* `RequestQueue` is used by {@apilink BasicCrawler}, {@apilink CheerioCrawler}, {@apilink PuppeteerCrawler}
|
|
79
|
+
* and {@apilink PlaywrightCrawler} as a source of URLs to crawl.
|
|
80
|
+
* Unlike {@apilink RequestList}, `RequestQueue` supports dynamic adding and removing of requests.
|
|
81
81
|
* On the other hand, the queue is not optimized for operations that add or remove a large number of URLs in a batch.
|
|
82
82
|
*
|
|
83
83
|
* `RequestQueue` stores its data either on local disk or in the Apify Cloud,
|
|
@@ -89,7 +89,7 @@ export interface RequestQueueOperationOptions {
|
|
|
89
89
|
* If the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the
|
|
90
90
|
* [Apify Request Queue](https://docs.apify.com/storage/request-queue)
|
|
91
91
|
* cloud storage. Note that you can force usage of the cloud storage also by passing the `forceCloud`
|
|
92
|
-
* option to {@
|
|
92
|
+
* option to {@apilink RequestQueue.open} function,
|
|
93
93
|
* even if the `APIFY_LOCAL_STORAGE_DIR` variable is set.
|
|
94
94
|
*
|
|
95
95
|
* **Example usage:**
|
|
@@ -152,12 +152,12 @@ export declare class RequestQueue {
|
|
|
152
152
|
*
|
|
153
153
|
* If a request with the same `uniqueKey` property is already present in the queue,
|
|
154
154
|
* it will not be updated. You can find out whether this happened from the resulting
|
|
155
|
-
* {@
|
|
155
|
+
* {@apilink QueueOperationInfo} object.
|
|
156
156
|
*
|
|
157
157
|
* To add multiple requests to the queue by extracting links from a webpage,
|
|
158
|
-
* see the {@
|
|
158
|
+
* see the {@apilink enqueueLinks} helper function.
|
|
159
159
|
*
|
|
160
|
-
* @param requestLike {@
|
|
160
|
+
* @param requestLike {@apilink Request} object or vanilla object with request data.
|
|
161
161
|
* Note that the function sets the `uniqueKey` and `id` fields to the passed Request.
|
|
162
162
|
* @param [options] Request queue operation options.
|
|
163
163
|
*/
|
|
@@ -167,9 +167,9 @@ export declare class RequestQueue {
|
|
|
167
167
|
*
|
|
168
168
|
* If a request that is passed in is already present due to its `uniqueKey` property being the same,
|
|
169
169
|
* it will not be updated. You can find out whether this happened by finding the request in the resulting
|
|
170
|
-
* {@
|
|
170
|
+
* {@apilink BatchAddRequestsResult} object.
|
|
171
171
|
*
|
|
172
|
-
* @param requestsLike {@
|
|
172
|
+
* @param requestsLike {@apilink Request} objects or vanilla objects with request data.
|
|
173
173
|
* Note that the function sets the `uniqueKey` and `id` fields to the passed requests if missing.
|
|
174
174
|
* @param [options] Request queue operation options.
|
|
175
175
|
*/
|
|
@@ -185,15 +185,15 @@ export declare class RequestQueue {
|
|
|
185
185
|
* Returns a next request in the queue to be processed, or `null` if there are no more pending requests.
|
|
186
186
|
*
|
|
187
187
|
* Once you successfully finish processing of the request, you need to call
|
|
188
|
-
* {@
|
|
188
|
+
* {@apilink RequestQueue.markRequestHandled}
|
|
189
189
|
* to mark the request as handled in the queue. If there was some error in processing the request,
|
|
190
|
-
* call {@
|
|
190
|
+
* call {@apilink RequestQueue.reclaimRequest} instead,
|
|
191
191
|
* so that the queue will give the request to some other consumer in another call to the `fetchNextRequest` function.
|
|
192
192
|
*
|
|
193
193
|
* Note that the `null` return value doesn't mean the queue processing finished,
|
|
194
194
|
* it means there are currently no pending requests.
|
|
195
195
|
* To check whether all requests in queue were finished,
|
|
196
|
-
* use {@
|
|
196
|
+
* use {@apilink RequestQueue.isFinished} instead.
|
|
197
197
|
*
|
|
198
198
|
* @returns
|
|
199
199
|
* Returns the request object or `null` if there are no more pending requests.
|
|
@@ -201,23 +201,23 @@ export declare class RequestQueue {
|
|
|
201
201
|
fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
|
|
202
202
|
/**
|
|
203
203
|
* Marks a request that was previously returned by the
|
|
204
|
-
* {@
|
|
204
|
+
* {@apilink RequestQueue.fetchNextRequest}
|
|
205
205
|
* function as handled after successful processing.
|
|
206
206
|
* Handled requests will never again be returned by the `fetchNextRequest` function.
|
|
207
207
|
*/
|
|
208
208
|
markRequestHandled(request: Request): Promise<QueueOperationInfo | null>;
|
|
209
209
|
/**
|
|
210
210
|
* Reclaims a failed request back to the queue, so that it can be returned for processing later again
|
|
211
|
-
* by another call to {@
|
|
211
|
+
* by another call to {@apilink RequestQueue.fetchNextRequest}.
|
|
212
212
|
* The request record in the queue is updated using the provided `request` parameter.
|
|
213
213
|
* For example, this lets you store the number of retries or error messages for the request.
|
|
214
214
|
*/
|
|
215
215
|
reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<QueueOperationInfo | null>;
|
|
216
216
|
/**
|
|
217
|
-
* Resolves to `true` if the next call to {@
|
|
217
|
+
* Resolves to `true` if the next call to {@apilink RequestQueue.fetchNextRequest}
|
|
218
218
|
* would return `null`, otherwise it resolves to `false`.
|
|
219
219
|
* Note that even if the queue is empty, there might be some pending requests currently being processed.
|
|
220
|
-
* If you need to ensure that there is no activity in the queue, use {@
|
|
220
|
+
* If you need to ensure that there is no activity in the queue, use {@apilink RequestQueue.isFinished}.
|
|
221
221
|
*/
|
|
222
222
|
isEmpty(): Promise<boolean>;
|
|
223
223
|
/**
|
|
@@ -290,14 +290,14 @@ export declare class RequestQueue {
|
|
|
290
290
|
getInfo(): Promise<RequestQueueInfo | undefined>;
|
|
291
291
|
/**
|
|
292
292
|
* Opens a request queue and returns a promise resolving to an instance
|
|
293
|
-
* of the {@
|
|
293
|
+
* of the {@apilink RequestQueue} class.
|
|
294
294
|
*
|
|
295
|
-
* {@
|
|
295
|
+
* {@apilink RequestQueue} represents a queue of URLs to crawl, which is stored either on local filesystem or in the cloud.
|
|
296
296
|
* The queue is used for deep crawling of websites, where you start with several URLs and then
|
|
297
297
|
* recursively follow links to other pages. The data structure supports both breadth-first
|
|
298
298
|
* and depth-first crawling orders.
|
|
299
299
|
*
|
|
300
|
-
* For more details and code examples, see the {@
|
|
300
|
+
* For more details and code examples, see the {@apilink RequestQueue} class.
|
|
301
301
|
*
|
|
302
302
|
* @param [queueIdOrName]
|
|
303
303
|
* ID or name of the request queue to be opened. If `null` or `undefined`,
|