@crawlee/core 3.0.3-beta.7 → 3.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/README.md +1 -1
  2. package/autoscaling/autoscaled_pool.d.ts +16 -16
  3. package/autoscaling/autoscaled_pool.js +13 -13
  4. package/autoscaling/snapshotter.d.ts +1 -1
  5. package/autoscaling/snapshotter.js +1 -1
  6. package/autoscaling/system_status.d.ts +12 -12
  7. package/autoscaling/system_status.js +11 -11
  8. package/configuration.d.ts +33 -13
  9. package/configuration.d.ts.map +1 -1
  10. package/configuration.js +27 -7
  11. package/configuration.js.map +1 -1
  12. package/crawlers/crawler_commons.d.ts +2 -2
  13. package/crawlers/statistics.d.ts +1 -1
  14. package/crawlers/statistics.js +1 -1
  15. package/enqueue_links/enqueue_links.d.ts +14 -13
  16. package/enqueue_links/enqueue_links.d.ts.map +1 -1
  17. package/enqueue_links/enqueue_links.js +6 -5
  18. package/enqueue_links/enqueue_links.js.map +1 -1
  19. package/enqueue_links/shared.d.ts +2 -2
  20. package/errors.d.ts +6 -0
  21. package/errors.d.ts.map +1 -1
  22. package/errors.js +10 -1
  23. package/errors.js.map +1 -1
  24. package/index.mjs +1 -0
  25. package/package.json +4 -4
  26. package/proxy_configuration.d.ts +11 -11
  27. package/proxy_configuration.js +8 -8
  28. package/request.d.ts +3 -3
  29. package/request.js +2 -2
  30. package/session_pool/session.d.ts +1 -1
  31. package/session_pool/session_pool.d.ts +12 -12
  32. package/session_pool/session_pool.d.ts.map +1 -1
  33. package/session_pool/session_pool.js +10 -10
  34. package/storages/dataset.d.ts +14 -14
  35. package/storages/dataset.js +8 -8
  36. package/storages/key_value_store.d.ts +30 -49
  37. package/storages/key_value_store.d.ts.map +1 -1
  38. package/storages/key_value_store.js +51 -46
  39. package/storages/key_value_store.js.map +1 -1
  40. package/storages/request_list.d.ts +33 -33
  41. package/storages/request_list.js +23 -23
  42. package/storages/request_queue.d.ts +25 -25
  43. package/storages/request_queue.js +23 -23
  44. package/storages/utils.d.ts +1 -1
  45. package/storages/utils.js +1 -1
  46. package/tsconfig.build.tsbuildinfo +1 -1
@@ -9,8 +9,8 @@ export declare const STATE_PERSISTENCE_KEY = "REQUEST_LIST_STATE";
9
9
  export declare const REQUESTS_PERSISTENCE_KEY = "REQUEST_LIST_REQUESTS";
10
10
  export interface RequestListOptions {
11
11
  /**
12
- * An array of sources of URLs for the {@link RequestList}. It can be either an array of strings,
13
- * plain objects that define at least the `url` property, or an array of {@link Request} instances.
12
+ * An array of sources of URLs for the {@apilink RequestList}. It can be either an array of strings,
13
+ * plain objects that define at least the `url` property, or an array of {@apilink Request} instances.
14
14
  *
15
15
  * **IMPORTANT:** The `sources` array will be consumed (left empty) after `RequestList` initializes.
16
16
  * This is a measure to prevent memory leaks in situations when millions of sources are
@@ -44,15 +44,15 @@ export interface RequestListOptions {
44
44
  sources?: Source[];
45
45
  /**
46
46
  * A function that will be called to get the sources for the `RequestList`, but only if `RequestList`
47
- * was not able to fetch their persisted version (see {@link RequestListOptions.persistRequestsKey}).
48
- * It must return an `Array` of {@link Request} or {@link RequestOptions}.
47
+ * was not able to fetch their persisted version (see {@apilink RequestListOptions.persistRequestsKey}).
48
+ * It must return an `Array` of {@apilink Request} or {@apilink RequestOptions}.
49
49
  *
50
50
  * This is very useful in a scenario when getting the sources is a resource intensive or time consuming
51
51
  * task, such as fetching URLs from multiple sitemaps or parsing URLs from large datasets. Using the
52
52
  * `sourcesFunction` in combination with `persistStateKey` and `persistRequestsKey` will allow you to
53
53
  * fetch and parse those URLs only once, saving valuable time when your crawler migrates or restarts.
54
54
  *
55
- * If both {@link RequestListOptions.sources} and {@link RequestListOptions.sourcesFunction} are provided,
55
+ * If both {@apilink RequestListOptions.sources} and {@apilink RequestListOptions.sourcesFunction} are provided,
56
56
  * the sources returned by the function will be added after the `sources`.
57
57
  *
58
58
  * **Example:**
@@ -99,12 +99,12 @@ export interface RequestListOptions {
99
99
  persistStateKey?: string;
100
100
  /**
101
101
  * Identifies the key in the default key-value store under which the `RequestList` persists its
102
- * Requests during the {@link RequestList.initialize} call.
102
+ * Requests during the {@apilink RequestList.initialize} call.
103
103
  * This is necessary if `persistStateKey` is set and the source URLs might potentially change,
104
104
  * to ensure consistency of the source URLs and state object. However, it comes with some
105
105
  * storage and performance overheads.
106
106
  *
107
- * If `persistRequestsKey` is not set, {@link RequestList.initialize} will always fetch the sources
107
+ * If `persistRequestsKey` is not set, {@apilink RequestList.initialize} will always fetch the sources
108
108
  * from their origin, check that they are consistent with the restored state (if any)
109
109
  * and throw an error if they are not.
110
110
  */
@@ -130,7 +130,7 @@ export interface RequestListOptions {
130
130
  state?: RequestListState;
131
131
  /**
132
132
  * By default, `RequestList` will deduplicate the provided URLs. Default deduplication is based
133
- * on the `uniqueKey` property of passed source {@link Request} objects.
133
+ * on the `uniqueKey` property of passed source {@apilink Request} objects.
134
134
  *
135
135
  * If the property is not present, it is generated by normalizing the URL. If present, it is kept intact.
136
136
  * In any case, only one request per `uniqueKey` is added to the `RequestList` resulting in removal
@@ -150,19 +150,19 @@ export interface RequestListOptions {
150
150
  /**
151
151
  * Represents a static list of URLs to crawl.
152
152
  * The URLs can be provided either in code or parsed from a text file hosted on the web.
153
- * `RequestList` is used by {@link BasicCrawler}, {@link CheerioCrawler}, {@link PuppeteerCrawler}
154
- * and {@link PlaywrightCrawler} as a source of URLs to crawl.
153
+ * `RequestList` is used by {@apilink BasicCrawler}, {@apilink CheerioCrawler}, {@apilink PuppeteerCrawler}
154
+ * and {@apilink PlaywrightCrawler} as a source of URLs to crawl.
155
155
  *
156
- * Each URL is represented using an instance of the {@link Request} class.
156
+ * Each URL is represented using an instance of the {@apilink Request} class.
157
157
  * The list can only contain unique URLs. More precisely, it can only contain `Request` instances
158
158
  * with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
159
- * To add a single URL to the list multiple times, corresponding {@link Request} objects will need to have different
159
+ * To add a single URL to the list multiple times, corresponding {@apilink Request} objects will need to have different
160
160
  * `uniqueKey` properties. You can use the `keepDuplicateUrls` option to do this for you when initializing the
161
161
  * `RequestList` from sources.
162
162
  *
163
- * Once you create an instance of `RequestList`, you need to call the {@link RequestList.initialize} function
163
+ * Once you create an instance of `RequestList`, you need to call the {@apilink RequestList.initialize} function
164
164
  * before the instance can be used. After that, no more URLs can be added to the list.
165
- * Unlike {@link RequestQueue}, `RequestList` is static but it can contain even millions of URLs.
165
+ * Unlike {@apilink RequestQueue}, `RequestList` is static but it can contain even millions of URLs.
166
166
  * > Note that `RequestList` can be used together with `RequestQueue` by the same crawler.
167
167
  * > In such cases, each request from `RequestList` is enqueued into `RequestQueue` first and then consumed from the latter.
168
168
  * > This is necessary to avoid the same URL being processed more than once (from the list first and then possibly from the queue).
@@ -171,9 +171,9 @@ export interface RequestListOptions {
171
171
  *
172
172
  * `RequestList` has an internal state where it stores information about which requests were already handled,
173
173
  * which are in progress and which were reclaimed. The state may be automatically persisted to the default
174
- * {@link KeyValueStore} by setting the `persistStateKey` option so that if the Node.js process is restarted,
174
+ * {@apilink KeyValueStore} by setting the `persistStateKey` option so that if the Node.js process is restarted,
175
175
  * the crawling can continue where it left off. The automated persisting is launched upon receiving the `persistState`
176
- * event that is periodically emitted by {@link EventManager}.
176
+ * event that is periodically emitted by {@apilink EventManager}.
177
177
  *
178
178
  * The internal state is closely tied to the provided sources (URLs). If the sources change on crawler restart, the state will become corrupted and
179
179
  * `RequestList` will raise an exception. This typically happens when the sources is a list of URLs downloaded from the web.
@@ -279,7 +279,7 @@ export declare class RequestList {
279
279
  */
280
280
  protected _addRequestsFromSources(): Promise<void>;
281
281
  /**
282
- * Persists the current state of the `RequestList` into the default {@link KeyValueStore}.
282
+ * Persists the current state of the `RequestList` into the default {@apilink KeyValueStore}.
283
283
  * The state is persisted automatically in regular intervals, but calling this method manually
284
284
  * is useful in cases where you want to have the most current state available after you pause
285
285
  * or stop fetching its requests. For example after you pause or abort a crawl. Or just before
@@ -307,7 +307,7 @@ export declare class RequestList {
307
307
  */
308
308
  getState(): RequestListState;
309
309
  /**
310
- * Resolves to `true` if the next call to {@link RequestList.fetchNextRequest} function
310
+ * Resolves to `true` if the next call to {@apilink RequestList.fetchNextRequest} function
311
311
  * would return `null`, otherwise it resolves to `false`.
312
312
  * Note that even if the list is empty, there might be some pending requests currently being processed.
313
313
  */
@@ -317,8 +317,8 @@ export declare class RequestList {
317
317
  */
318
318
  isFinished(): Promise<boolean>;
319
319
  /**
320
- * Gets the next {@link Request} to process. First, the function gets a request previously reclaimed
321
- * using the {@link RequestList.reclaimRequest} function, if there is any.
320
+ * Gets the next {@apilink Request} to process. First, the function gets a request previously reclaimed
321
+ * using the {@apilink RequestList.reclaimRequest} function, if there is any.
322
322
  * Otherwise it gets the next request from sources.
323
323
  *
324
324
  * The function's `Promise` resolves to `null` if there are no more
@@ -372,14 +372,14 @@ export declare class RequestList {
372
372
  handledCount(): number;
373
373
  /**
374
374
  * Opens a request list and returns a promise resolving to an instance
375
- * of the {@link RequestList} class that is already initialized.
375
+ * of the {@apilink RequestList} class that is already initialized.
376
376
  *
377
- * {@link RequestList} represents a list of URLs to crawl, which is always stored in memory.
377
+ * {@apilink RequestList} represents a list of URLs to crawl, which is always stored in memory.
378
378
  * To enable picking up where left off after a process restart, the request list sources
379
379
  * are persisted to the key-value store at initialization of the list. Then, while crawling,
380
380
  * a small state object is regularly persisted to keep track of the crawling status.
381
381
  *
382
- * For more details and code examples, see the {@link RequestList} class.
382
+ * For more details and code examples, see the {@apilink RequestList} class.
383
383
  *
384
384
  * **Example usage:**
385
385
  *
@@ -405,23 +405,23 @@ export declare class RequestList {
405
405
  * If `null`, the list will not be persisted and will only be stored in memory. Process restart
406
406
  * will then cause the list to be crawled again from the beginning. We suggest always using a name.
407
407
  * @param [sources]
408
- * An array of sources of URLs for the {@link RequestList}. It can be either an array of strings,
409
- * plain objects that define at least the `url` property, or an array of {@link Request} instances.
408
+ * An array of sources of URLs for the {@apilink RequestList}. It can be either an array of strings,
409
+ * plain objects that define at least the `url` property, or an array of {@apilink Request} instances.
410
410
  *
411
- * **IMPORTANT:** The `sources` array will be consumed (left empty) after {@link RequestList} initializes.
411
+ * **IMPORTANT:** The `sources` array will be consumed (left empty) after {@apilink RequestList} initializes.
412
412
  * This is a measure to prevent memory leaks in situations when millions of sources are
413
413
  * added.
414
414
  *
415
415
  * Additionally, the `requestsFromUrl` property may be used instead of `url`,
416
- * which will instruct {@link RequestList} to download the source URLs from a given remote location.
416
+ * which will instruct {@apilink RequestList} to download the source URLs from a given remote location.
417
417
  * The URLs will be parsed from the received response. In this case you can limit the URLs
418
418
  * using `regex` parameter containing regular expression pattern for URLs to be included.
419
419
  *
420
- * For details, see the {@link RequestListOptions.sources}
420
+ * For details, see the {@apilink RequestListOptions.sources}
421
421
  * @param [options]
422
- * The {@link RequestList} options. Note that the `listName` parameter supersedes
423
- * the {@link RequestListOptions.persistStateKey} and {@link RequestListOptions.persistRequestsKey}
424
- * options and the `sources` parameter supersedes the {@link RequestListOptions.sources} option.
422
+ * The {@apilink RequestList} options. Note that the `listName` parameter supersedes
423
+ * the {@apilink RequestListOptions.persistStateKey} and {@apilink RequestListOptions.persistRequestsKey}
424
+ * options and the `sources` parameter supersedes the {@apilink RequestListOptions.sources} option.
425
425
  */
426
426
  static open(listNameOrOptions: string | null | RequestListOptions, sources?: Source[], options?: RequestListOptions): Promise<RequestList>;
427
427
  /**
@@ -430,8 +430,8 @@ export declare class RequestList {
430
430
  private _downloadListOfUrls;
431
431
  }
432
432
  /**
433
- * Represents state of a {@link RequestList}. It can be used to resume a {@link RequestList} which has been previously processed.
434
- * You can obtain the state by calling {@link RequestList.getState} and receive an object with
433
+ * Represents state of a {@apilink RequestList}. It can be used to resume a {@apilink RequestList} which has been previously processed.
434
+ * You can obtain the state by calling {@apilink RequestList.getState} and receive an object with
435
435
  * the following structure:
436
436
  *
437
437
  * ```
@@ -18,19 +18,19 @@ const CONTENT_TYPE_BINARY = 'application/octet-stream';
18
18
  /**
19
19
  * Represents a static list of URLs to crawl.
20
20
  * The URLs can be provided either in code or parsed from a text file hosted on the web.
21
- * `RequestList` is used by {@link BasicCrawler}, {@link CheerioCrawler}, {@link PuppeteerCrawler}
22
- * and {@link PlaywrightCrawler} as a source of URLs to crawl.
21
+ * `RequestList` is used by {@apilink BasicCrawler}, {@apilink CheerioCrawler}, {@apilink PuppeteerCrawler}
22
+ * and {@apilink PlaywrightCrawler} as a source of URLs to crawl.
23
23
  *
24
- * Each URL is represented using an instance of the {@link Request} class.
24
+ * Each URL is represented using an instance of the {@apilink Request} class.
25
25
  * The list can only contain unique URLs. More precisely, it can only contain `Request` instances
26
26
  * with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
27
- * To add a single URL to the list multiple times, corresponding {@link Request} objects will need to have different
27
+ * To add a single URL to the list multiple times, corresponding {@apilink Request} objects will need to have different
28
28
  * `uniqueKey` properties. You can use the `keepDuplicateUrls` option to do this for you when initializing the
29
29
  * `RequestList` from sources.
30
30
  *
31
- * Once you create an instance of `RequestList`, you need to call the {@link RequestList.initialize} function
31
+ * Once you create an instance of `RequestList`, you need to call the {@apilink RequestList.initialize} function
32
32
  * before the instance can be used. After that, no more URLs can be added to the list.
33
- * Unlike {@link RequestQueue}, `RequestList` is static but it can contain even millions of URLs.
33
+ * Unlike {@apilink RequestQueue}, `RequestList` is static but it can contain even millions of URLs.
34
34
  * > Note that `RequestList` can be used together with `RequestQueue` by the same crawler.
35
35
  * > In such cases, each request from `RequestList` is enqueued into `RequestQueue` first and then consumed from the latter.
36
36
  * > This is necessary to avoid the same URL being processed more than once (from the list first and then possibly from the queue).
@@ -39,9 +39,9 @@ const CONTENT_TYPE_BINARY = 'application/octet-stream';
39
39
  *
40
40
  * `RequestList` has an internal state where it stores information about which requests were already handled,
41
41
  * which are in progress and which were reclaimed. The state may be automatically persisted to the default
42
- * {@link KeyValueStore} by setting the `persistStateKey` option so that if the Node.js process is restarted,
42
+ * {@apilink KeyValueStore} by setting the `persistStateKey` option so that if the Node.js process is restarted,
43
43
  * the crawling can continue where it left off. The automated persisting is launched upon receiving the `persistState`
44
- * event that is periodically emitted by {@link EventManager}.
44
+ * event that is periodically emitted by {@apilink EventManager}.
45
45
  *
46
46
  * The internal state is closely tied to the provided sources (URLs). If the sources change on crawler restart, the state will become corrupted and
47
47
  * `RequestList` will raise an exception. This typically happens when the sources is a list of URLs downloaded from the web.
@@ -338,7 +338,7 @@ class RequestList {
338
338
  }
339
339
  }
340
340
  /**
341
- * Persists the current state of the `RequestList` into the default {@link KeyValueStore}.
341
+ * Persists the current state of the `RequestList` into the default {@apilink KeyValueStore}.
342
342
  * The state is persisted automatically in regular intervals, but calling this method manually
343
343
  * is useful in cases where you want to have the most current state available after you pause
344
344
  * or stop fetching its requests. For example after you pause or abort a crawl. Or just before
@@ -463,7 +463,7 @@ class RequestList {
463
463
  };
464
464
  }
465
465
  /**
466
- * Resolves to `true` if the next call to {@link RequestList.fetchNextRequest} function
466
+ * Resolves to `true` if the next call to {@apilink RequestList.fetchNextRequest} function
467
467
  * would return `null`, otherwise it resolves to `false`.
468
468
  * Note that even if the list is empty, there might be some pending requests currently being processed.
469
469
  */
@@ -479,8 +479,8 @@ class RequestList {
479
479
  return this.inProgress.size === 0 && this.nextIndex >= this.requests.length;
480
480
  }
481
481
  /**
482
- * Gets the next {@link Request} to process. First, the function gets a request previously reclaimed
483
- * using the {@link RequestList.reclaimRequest} function, if there is any.
482
+ * Gets the next {@apilink Request} to process. First, the function gets a request previously reclaimed
483
+ * using the {@apilink RequestList.reclaimRequest} function, if there is any.
484
484
  * Otherwise it gets the next request from sources.
485
485
  *
486
486
  * The function's `Promise` resolves to `null` if there are no more
@@ -650,14 +650,14 @@ class RequestList {
650
650
  }
651
651
  /**
652
652
  * Opens a request list and returns a promise resolving to an instance
653
- * of the {@link RequestList} class that is already initialized.
653
+ * of the {@apilink RequestList} class that is already initialized.
654
654
  *
655
- * {@link RequestList} represents a list of URLs to crawl, which is always stored in memory.
655
+ * {@apilink RequestList} represents a list of URLs to crawl, which is always stored in memory.
656
656
  * To enable picking up where left off after a process restart, the request list sources
657
657
  * are persisted to the key-value store at initialization of the list. Then, while crawling,
658
658
  * a small state object is regularly persisted to keep track of the crawling status.
659
659
  *
660
- * For more details and code examples, see the {@link RequestList} class.
660
+ * For more details and code examples, see the {@apilink RequestList} class.
661
661
  *
662
662
  * **Example usage:**
663
663
  *
@@ -683,23 +683,23 @@ class RequestList {
683
683
  * If `null`, the list will not be persisted and will only be stored in memory. Process restart
684
684
  * will then cause the list to be crawled again from the beginning. We suggest always using a name.
685
685
  * @param [sources]
686
- * An array of sources of URLs for the {@link RequestList}. It can be either an array of strings,
687
- * plain objects that define at least the `url` property, or an array of {@link Request} instances.
686
+ * An array of sources of URLs for the {@apilink RequestList}. It can be either an array of strings,
687
+ * plain objects that define at least the `url` property, or an array of {@apilink Request} instances.
688
688
  *
689
- * **IMPORTANT:** The `sources` array will be consumed (left empty) after {@link RequestList} initializes.
689
+ * **IMPORTANT:** The `sources` array will be consumed (left empty) after {@apilink RequestList} initializes.
690
690
  * This is a measure to prevent memory leaks in situations when millions of sources are
691
691
  * added.
692
692
  *
693
693
  * Additionally, the `requestsFromUrl` property may be used instead of `url`,
694
- * which will instruct {@link RequestList} to download the source URLs from a given remote location.
694
+ * which will instruct {@apilink RequestList} to download the source URLs from a given remote location.
695
695
  * The URLs will be parsed from the received response. In this case you can limit the URLs
696
696
  * using `regex` parameter containing regular expression pattern for URLs to be included.
697
697
  *
698
- * For details, see the {@link RequestListOptions.sources}
698
+ * For details, see the {@apilink RequestListOptions.sources}
699
699
  * @param [options]
700
- * The {@link RequestList} options. Note that the `listName` parameter supersedes
701
- * the {@link RequestListOptions.persistStateKey} and {@link RequestListOptions.persistRequestsKey}
702
- * options and the `sources` parameter supersedes the {@link RequestListOptions.sources} option.
700
+ * The {@apilink RequestList} options. Note that the `listName` parameter supersedes
701
+ * the {@apilink RequestListOptions.persistStateKey} and {@apilink RequestListOptions.persistRequestsKey}
702
+ * options and the `sources` parameter supersedes the {@apilink RequestListOptions.sources} option.
703
703
  */
704
704
  static async open(listNameOrOptions, sources, options = {}) {
705
705
  if (listNameOrOptions != null && typeof listNameOrOptions === 'object') {
@@ -40,7 +40,7 @@ export declare const STORAGE_CONSISTENCY_DELAY_MILLIS = 3000;
40
40
  export declare function getRequestId(uniqueKey: string): string;
41
41
  /**
42
42
  * A helper class that is used to report results from various
43
- * {@link RequestQueue} functions as well as {@link enqueueLinks}.
43
+ * {@apilink RequestQueue} functions as well as {@apilink enqueueLinks}.
44
44
  */
45
45
  export interface QueueOperationInfo {
46
46
  /** Indicates if request was already present in the queue. */
@@ -56,7 +56,7 @@ export interface RequestQueueOperationOptions {
56
56
  * If set to `true`:
57
57
  * - while adding the request to the queue: the request will be added to the foremost position in the queue.
58
58
  * - while reclaiming the request: the request will be placed to the beginning of the queue, so that it's returned
59
- * in the next call to {@link RequestQueue.fetchNextRequest}.
59
+ * in the next call to {@apilink RequestQueue.fetchNextRequest}.
60
60
  * By default, it's put to the end of the queue.
61
61
  * @default false
62
62
  */
@@ -67,17 +67,17 @@ export interface RequestQueueOperationOptions {
67
67
  * where you start with several URLs and then recursively
68
68
  * follow links to other pages. The data structure supports both breadth-first and depth-first crawling orders.
69
69
  *
70
- * Each URL is represented using an instance of the {@link Request} class.
71
- * The queue can only contain unique URLs. More precisely, it can only contain {@link Request} instances
70
+ * Each URL is represented using an instance of the {@apilink Request} class.
71
+ * The queue can only contain unique URLs. More precisely, it can only contain {@apilink Request} instances
72
72
  * with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
73
73
  * To add a single URL multiple times to the queue,
74
- * corresponding {@link Request} objects will need to have different `uniqueKey` properties.
74
+ * corresponding {@apilink Request} objects will need to have different `uniqueKey` properties.
75
75
  *
76
- * Do not instantiate this class directly, use the {@link RequestQueue.open} function instead.
76
+ * Do not instantiate this class directly, use the {@apilink RequestQueue.open} function instead.
77
77
  *
78
- * `RequestQueue` is used by {@link BasicCrawler}, {@link CheerioCrawler}, {@link PuppeteerCrawler}
79
- * and {@link PlaywrightCrawler} as a source of URLs to crawl.
80
- * Unlike {@link RequestList}, `RequestQueue` supports dynamic adding and removing of requests.
78
+ * `RequestQueue` is used by {@apilink BasicCrawler}, {@apilink CheerioCrawler}, {@apilink PuppeteerCrawler}
79
+ * and {@apilink PlaywrightCrawler} as a source of URLs to crawl.
80
+ * Unlike {@apilink RequestList}, `RequestQueue` supports dynamic adding and removing of requests.
81
81
  * On the other hand, the queue is not optimized for operations that add or remove a large number of URLs in a batch.
82
82
  *
83
83
  * `RequestQueue` stores its data either on local disk or in the Apify Cloud,
@@ -89,7 +89,7 @@ export interface RequestQueueOperationOptions {
89
89
  * If the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the
90
90
  * [Apify Request Queue](https://docs.apify.com/storage/request-queue)
91
91
  * cloud storage. Note that you can force usage of the cloud storage also by passing the `forceCloud`
92
- * option to {@link RequestQueue.open} function,
92
+ * option to {@apilink RequestQueue.open} function,
93
93
  * even if the `APIFY_LOCAL_STORAGE_DIR` variable is set.
94
94
  *
95
95
  * **Example usage:**
@@ -152,12 +152,12 @@ export declare class RequestQueue {
152
152
  *
153
153
  * If a request with the same `uniqueKey` property is already present in the queue,
154
154
  * it will not be updated. You can find out whether this happened from the resulting
155
- * {@link QueueOperationInfo} object.
155
+ * {@apilink QueueOperationInfo} object.
156
156
  *
157
157
  * To add multiple requests to the queue by extracting links from a webpage,
158
- * see the {@link enqueueLinks} helper function.
158
+ * see the {@apilink enqueueLinks} helper function.
159
159
  *
160
- * @param requestLike {@link Request} object or vanilla object with request data.
160
+ * @param requestLike {@apilink Request} object or vanilla object with request data.
161
161
  * Note that the function sets the `uniqueKey` and `id` fields to the passed Request.
162
162
  * @param [options] Request queue operation options.
163
163
  */
@@ -167,9 +167,9 @@ export declare class RequestQueue {
167
167
  *
168
168
  * If a request that is passed in is already present due to its `uniqueKey` property being the same,
169
169
  * it will not be updated. You can find out whether this happened by finding the request in the resulting
170
- * {@link BatchAddRequestsResult} object.
170
+ * {@apilink BatchAddRequestsResult} object.
171
171
  *
172
- * @param requestsLike {@link Request} objects or vanilla objects with request data.
172
+ * @param requestsLike {@apilink Request} objects or vanilla objects with request data.
173
173
  * Note that the function sets the `uniqueKey` and `id` fields to the passed requests if missing.
174
174
  * @param [options] Request queue operation options.
175
175
  */
@@ -185,15 +185,15 @@ export declare class RequestQueue {
185
185
  * Returns a next request in the queue to be processed, or `null` if there are no more pending requests.
186
186
  *
187
187
  * Once you successfully finish processing of the request, you need to call
188
- * {@link RequestQueue.markRequestHandled}
188
+ * {@apilink RequestQueue.markRequestHandled}
189
189
  * to mark the request as handled in the queue. If there was some error in processing the request,
190
- * call {@link RequestQueue.reclaimRequest} instead,
190
+ * call {@apilink RequestQueue.reclaimRequest} instead,
191
191
  * so that the queue will give the request to some other consumer in another call to the `fetchNextRequest` function.
192
192
  *
193
193
  * Note that the `null` return value doesn't mean the queue processing finished,
194
194
  * it means there are currently no pending requests.
195
195
  * To check whether all requests in queue were finished,
196
- * use {@link RequestQueue.isFinished} instead.
196
+ * use {@apilink RequestQueue.isFinished} instead.
197
197
  *
198
198
  * @returns
199
199
  * Returns the request object or `null` if there are no more pending requests.
@@ -201,23 +201,23 @@ export declare class RequestQueue {
201
201
  fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
202
202
  /**
203
203
  * Marks a request that was previously returned by the
204
- * {@link RequestQueue.fetchNextRequest}
204
+ * {@apilink RequestQueue.fetchNextRequest}
205
205
  * function as handled after successful processing.
206
206
  * Handled requests will never again be returned by the `fetchNextRequest` function.
207
207
  */
208
208
  markRequestHandled(request: Request): Promise<QueueOperationInfo | null>;
209
209
  /**
210
210
  * Reclaims a failed request back to the queue, so that it can be returned for processing later again
211
- * by another call to {@link RequestQueue.fetchNextRequest}.
211
+ * by another call to {@apilink RequestQueue.fetchNextRequest}.
212
212
  * The request record in the queue is updated using the provided `request` parameter.
213
213
  * For example, this lets you store the number of retries or error messages for the request.
214
214
  */
215
215
  reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<QueueOperationInfo | null>;
216
216
  /**
217
- * Resolves to `true` if the next call to {@link RequestQueue.fetchNextRequest}
217
+ * Resolves to `true` if the next call to {@apilink RequestQueue.fetchNextRequest}
218
218
  * would return `null`, otherwise it resolves to `false`.
219
219
  * Note that even if the queue is empty, there might be some pending requests currently being processed.
220
- * If you need to ensure that there is no activity in the queue, use {@link RequestQueue.isFinished}.
220
+ * If you need to ensure that there is no activity in the queue, use {@apilink RequestQueue.isFinished}.
221
221
  */
222
222
  isEmpty(): Promise<boolean>;
223
223
  /**
@@ -290,14 +290,14 @@ export declare class RequestQueue {
290
290
  getInfo(): Promise<RequestQueueInfo | undefined>;
291
291
  /**
292
292
  * Opens a request queue and returns a promise resolving to an instance
293
- * of the {@link RequestQueue} class.
293
+ * of the {@apilink RequestQueue} class.
294
294
  *
295
- * {@link RequestQueue} represents a queue of URLs to crawl, which is stored either on local filesystem or in the cloud.
295
+ * {@apilink RequestQueue} represents a queue of URLs to crawl, which is stored either on local filesystem or in the cloud.
296
296
  * The queue is used for deep crawling of websites, where you start with several URLs and then
297
297
  * recursively follow links to other pages. The data structure supports both breadth-first
298
298
  * and depth-first crawling orders.
299
299
  *
300
- * For more details and code examples, see the {@link RequestQueue} class.
300
+ * For more details and code examples, see the {@apilink RequestQueue} class.
301
301
  *
302
302
  * @param [queueIdOrName]
303
303
  * ID or name of the request queue to be opened. If `null` or `undefined`,
@@ -67,17 +67,17 @@ exports.getRequestId = getRequestId;
67
67
  * where you start with several URLs and then recursively
68
68
  * follow links to other pages. The data structure supports both breadth-first and depth-first crawling orders.
69
69
  *
70
- * Each URL is represented using an instance of the {@link Request} class.
71
- * The queue can only contain unique URLs. More precisely, it can only contain {@link Request} instances
70
+ * Each URL is represented using an instance of the {@apilink Request} class.
71
+ * The queue can only contain unique URLs. More precisely, it can only contain {@apilink Request} instances
72
72
  * with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
73
73
  * To add a single URL multiple times to the queue,
74
- * corresponding {@link Request} objects will need to have different `uniqueKey` properties.
74
+ * corresponding {@apilink Request} objects will need to have different `uniqueKey` properties.
75
75
  *
76
- * Do not instantiate this class directly, use the {@link RequestQueue.open} function instead.
76
+ * Do not instantiate this class directly, use the {@apilink RequestQueue.open} function instead.
77
77
  *
78
- * `RequestQueue` is used by {@link BasicCrawler}, {@link CheerioCrawler}, {@link PuppeteerCrawler}
79
- * and {@link PlaywrightCrawler} as a source of URLs to crawl.
80
- * Unlike {@link RequestList}, `RequestQueue` supports dynamic adding and removing of requests.
78
+ * `RequestQueue` is used by {@apilink BasicCrawler}, {@apilink CheerioCrawler}, {@apilink PuppeteerCrawler}
79
+ * and {@apilink PlaywrightCrawler} as a source of URLs to crawl.
80
+ * Unlike {@apilink RequestList}, `RequestQueue` supports dynamic adding and removing of requests.
81
81
  * On the other hand, the queue is not optimized for operations that add or remove a large number of URLs in a batch.
82
82
  *
83
83
  * `RequestQueue` stores its data either on local disk or in the Apify Cloud,
@@ -89,7 +89,7 @@ exports.getRequestId = getRequestId;
89
89
  * If the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the
90
90
  * [Apify Request Queue](https://docs.apify.com/storage/request-queue)
91
91
  * cloud storage. Note that you can force usage of the cloud storage also by passing the `forceCloud`
92
- * option to {@link RequestQueue.open} function,
92
+ * option to {@apilink RequestQueue.open} function,
93
93
  * even if the `APIFY_LOCAL_STORAGE_DIR` variable is set.
94
94
  *
95
95
  * **Example usage:**
@@ -244,12 +244,12 @@ class RequestQueue {
244
244
  *
245
245
  * If a request with the same `uniqueKey` property is already present in the queue,
246
246
  * it will not be updated. You can find out whether this happened from the resulting
247
- * {@link QueueOperationInfo} object.
247
+ * {@apilink QueueOperationInfo} object.
248
248
  *
249
249
  * To add multiple requests to the queue by extracting links from a webpage,
250
- * see the {@link enqueueLinks} helper function.
250
+ * see the {@apilink enqueueLinks} helper function.
251
251
  *
252
- * @param requestLike {@link Request} object or vanilla object with request data.
252
+ * @param requestLike {@apilink Request} object or vanilla object with request data.
253
253
  * Note that the function sets the `uniqueKey` and `id` fields to the passed Request.
254
254
  * @param [options] Request queue operation options.
255
255
  */
@@ -295,9 +295,9 @@ class RequestQueue {
295
295
  *
296
296
  * If a request that is passed in is already present due to its `uniqueKey` property being the same,
297
297
  * it will not be updated. You can find out whether this happened by finding the request in the resulting
298
- * {@link BatchAddRequestsResult} object.
298
+ * {@apilink BatchAddRequestsResult} object.
299
299
  *
300
- * @param requestsLike {@link Request} objects or vanilla objects with request data.
300
+ * @param requestsLike {@apilink Request} objects or vanilla objects with request data.
301
301
  * Note that the function sets the `uniqueKey` and `id` fields to the passed requests if missing.
302
302
  * @param [options] Request queue operation options.
303
303
  */
@@ -386,15 +386,15 @@ class RequestQueue {
386
386
  * Returns a next request in the queue to be processed, or `null` if there are no more pending requests.
387
387
  *
388
388
  * Once you successfully finish processing of the request, you need to call
389
- * {@link RequestQueue.markRequestHandled}
389
+ * {@apilink RequestQueue.markRequestHandled}
390
390
  * to mark the request as handled in the queue. If there was some error in processing the request,
391
- * call {@link RequestQueue.reclaimRequest} instead,
391
+ * call {@apilink RequestQueue.reclaimRequest} instead,
392
392
  * so that the queue will give the request to some other consumer in another call to the `fetchNextRequest` function.
393
393
  *
394
394
  * Note that the `null` return value doesn't mean the queue processing finished,
395
395
  * it means there are currently no pending requests.
396
396
  * To check whether all requests in queue were finished,
397
- * use {@link RequestQueue.isFinished} instead.
397
+ * use {@apilink RequestQueue.isFinished} instead.
398
398
  *
399
399
  * @returns
400
400
  * Returns the request object or `null` if there are no more pending requests.
@@ -451,7 +451,7 @@ class RequestQueue {
451
451
  }
452
452
  /**
453
453
  * Marks a request that was previously returned by the
454
- * {@link RequestQueue.fetchNextRequest}
454
+ * {@apilink RequestQueue.fetchNextRequest}
455
455
  * function as handled after successful processing.
456
456
  * Handled requests will never again be returned by the `fetchNextRequest` function.
457
457
  */
@@ -480,7 +480,7 @@ class RequestQueue {
480
480
  }
481
481
  /**
482
482
  * Reclaims a failed request back to the queue, so that it can be returned for processing later again
483
- * by another call to {@link RequestQueue.fetchNextRequest}.
483
+ * by another call to {@apilink RequestQueue.fetchNextRequest}.
484
484
  * The request record in the queue is updated using the provided `request` parameter.
485
485
  * For example, this lets you store the number of retries or error messages for the request.
486
486
  */
@@ -517,10 +517,10 @@ class RequestQueue {
517
517
  return queueOperationInfo;
518
518
  }
519
519
  /**
520
- * Resolves to `true` if the next call to {@link RequestQueue.fetchNextRequest}
520
+ * Resolves to `true` if the next call to {@apilink RequestQueue.fetchNextRequest}
521
521
  * would return `null`, otherwise it resolves to `false`.
522
522
  * Note that even if the queue is empty, there might be some pending requests currently being processed.
523
- * If you need to ensure that there is no activity in the queue, use {@link RequestQueue.isFinished}.
523
+ * If you need to ensure that there is no activity in the queue, use {@apilink RequestQueue.isFinished}.
524
524
  */
525
525
  async isEmpty() {
526
526
  await this._ensureHeadIsNonEmpty();
@@ -709,14 +709,14 @@ class RequestQueue {
709
709
  }
710
710
  /**
711
711
  * Opens a request queue and returns a promise resolving to an instance
712
- * of the {@link RequestQueue} class.
712
+ * of the {@apilink RequestQueue} class.
713
713
  *
714
- * {@link RequestQueue} represents a queue of URLs to crawl, which is stored either on local filesystem or in the cloud.
714
+ * {@apilink RequestQueue} represents a queue of URLs to crawl, which is stored either on local filesystem or in the cloud.
715
715
  * The queue is used for deep crawling of websites, where you start with several URLs and then
716
716
  * recursively follow links to other pages. The data structure supports both breadth-first
717
717
  * and depth-first crawling orders.
718
718
  *
719
- * For more details and code examples, see the {@link RequestQueue} class.
719
+ * For more details and code examples, see the {@apilink RequestQueue} class.
720
720
  *
721
721
  * @param [queueIdOrName]
722
722
  * ID or name of the request queue to be opened. If `null` or `undefined`,
@@ -4,7 +4,7 @@ import { Configuration } from '../configuration';
4
4
  * Purging will remove all the files in all storages except for INPUT.json in the default KV store.
5
5
  *
6
6
  * Purging of storages is happening automatically when we run our crawler (or when we open some storage
7
- * explicitly, e.g. via `RequestList.open()`). We can disable that via `purgeOnStart` {@link Configuration}
7
+ * explicitly, e.g. via `RequestList.open()`). We can disable that via `purgeOnStart` {@apilink Configuration}
8
8
  * option or by setting `CRAWLEE_PURGE_ON_START` environment variable to `0` or `false`.
9
9
  *
10
10
  * This is a shortcut for running (optional) `purge` method on the StorageClient interface, in other words
package/storages/utils.js CHANGED
@@ -7,7 +7,7 @@ const configuration_1 = require("../configuration");
7
7
  * Purging will remove all the files in all storages except for INPUT.json in the default KV store.
8
8
  *
9
9
  * Purging of storages is happening automatically when we run our crawler (or when we open some storage
10
- * explicitly, e.g. via `RequestList.open()`). We can disable that via `purgeOnStart` {@link Configuration}
10
+ * explicitly, e.g. via `RequestList.open()`). We can disable that via `purgeOnStart` {@apilink Configuration}
11
11
  * option or by setting `CRAWLEE_PURGE_ON_START` environment variable to `0` or `false`.
12
12
  *
13
13
  * This is a shortcut for running (optional) `purge` method on the StorageClient interface, in other words