@crawlee/basic 3.0.3-beta.2 → 3.0.3-beta.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/index.mjs +1 -0
- package/internals/basic-crawler.d.ts +85 -70
- package/internals/basic-crawler.d.ts.map +1 -1
- package/internals/basic-crawler.js +88 -45
- package/internals/basic-crawler.js.map +1 -1
- package/internals/constants.d.ts +2 -2
- package/internals/constants.js +2 -2
- package/package.json +4 -4
- package/tsconfig.build.tsbuildinfo +1 -1
package/README.md
CHANGED
|
@@ -63,7 +63,7 @@ Additionally, the package provides various helper functions to simplify running
|
|
|
63
63
|
|
|
64
64
|
## Quick Start
|
|
65
65
|
|
|
66
|
-
This short tutorial will set you up to start using Crawlee in a minute or two. If you want to learn more, proceed to the [Getting Started](https://crawlee.dev/docs/
|
|
66
|
+
This short tutorial will set you up to start using Crawlee in a minute or two. If you want to learn more, proceed to the [Getting Started](https://crawlee.dev/docs/introduction) tutorial that will take you step by step through creating your first scraper.
|
|
67
67
|
|
|
68
68
|
### Local stand-alone usage
|
|
69
69
|
|
package/index.mjs
CHANGED
|
@@ -33,6 +33,7 @@ export const REQUESTS_PERSISTENCE_KEY = mod.REQUESTS_PERSISTENCE_KEY;
|
|
|
33
33
|
export const Request = mod.Request;
|
|
34
34
|
export const RequestList = mod.RequestList;
|
|
35
35
|
export const RequestQueue = mod.RequestQueue;
|
|
36
|
+
export const RetryRequestError = mod.RetryRequestError;
|
|
36
37
|
export const Router = mod.Router;
|
|
37
38
|
export const STATE_PERSISTENCE_KEY = mod.STATE_PERSISTENCE_KEY;
|
|
38
39
|
export const STORAGE_CONSISTENCY_DELAY_MILLIS = mod.STORAGE_CONSISTENCY_DELAY_MILLIS;
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import type { Log } from '@apify/log';
|
|
2
2
|
import type { AutoscaledPoolOptions, EnqueueLinksOptions, EventManager, FinalStatistics, ProxyInfo, Request, RequestList, RequestOptions, RequestQueueOperationOptions, RouterHandler, Session, SessionPoolOptions } from '@crawlee/core';
|
|
3
3
|
import { AutoscaledPool, Configuration, type CrawlingContext, RequestQueue, SessionPool, Statistics } from '@crawlee/core';
|
|
4
|
-
import type {
|
|
4
|
+
import type { OptionsInit, Response as GotResponse } from 'got-scraping';
|
|
5
5
|
import type { ProcessedRequest, Dictionary, Awaitable, BatchAddRequestsResult } from '@crawlee/types';
|
|
6
6
|
export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
|
|
7
7
|
crawler: BasicCrawler;
|
|
8
8
|
enqueueLinks: (options: BasicCrawlerEnqueueLinksOptions) => Promise<BatchAddRequestsResult>;
|
|
9
|
-
sendRequest: (overrideOptions?: Partial<
|
|
9
|
+
sendRequest: (overrideOptions?: Partial<OptionsInit>) => Promise<GotResponse<string>>;
|
|
10
10
|
}
|
|
11
11
|
/** @internal */
|
|
12
12
|
export interface BasicCrawlerEnqueueLinksOptions extends Omit<EnqueueLinksOptions, 'requestQueue'> {
|
|
@@ -17,37 +17,37 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
|
|
|
17
17
|
/**
|
|
18
18
|
* User-provided function that performs the logic of the crawler. It is called for each URL to crawl.
|
|
19
19
|
*
|
|
20
|
-
* The function receives the {@
|
|
21
|
-
* where the {@
|
|
20
|
+
* The function receives the {@apilink BasicCrawlingContext} as an argument,
|
|
21
|
+
* where the {@apilink BasicCrawlingContext.request|`request`} represents the URL to crawl.
|
|
22
22
|
*
|
|
23
23
|
* The function must return a promise, which is then awaited by the crawler.
|
|
24
24
|
*
|
|
25
25
|
* If the function throws an exception, the crawler will try to re-crawl the
|
|
26
|
-
* request later, up to the {@
|
|
26
|
+
* request later, up to the {@apilink BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
|
|
27
27
|
* If all the retries fail, the crawler calls the function
|
|
28
|
-
* provided to the {@
|
|
28
|
+
* provided to the {@apilink BasicCrawlerOptions.failedRequestHandler|`failedRequestHandler`} parameter.
|
|
29
29
|
* To make this work, we should **always**
|
|
30
30
|
* let our function throw exceptions rather than catch them.
|
|
31
31
|
* The exceptions are logged to the request using the
|
|
32
|
-
* {@
|
|
32
|
+
* {@apilink Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
|
|
33
33
|
*/
|
|
34
34
|
requestHandler?: RequestHandler<Context>;
|
|
35
35
|
/**
|
|
36
36
|
* User-provided function that performs the logic of the crawler. It is called for each URL to crawl.
|
|
37
37
|
*
|
|
38
|
-
* The function receives the {@
|
|
39
|
-
* where the {@
|
|
38
|
+
* The function receives the {@apilink BasicCrawlingContext} as an argument,
|
|
39
|
+
* where the {@apilink BasicCrawlingContext.request|`request`} represents the URL to crawl.
|
|
40
40
|
*
|
|
41
41
|
* The function must return a promise, which is then awaited by the crawler.
|
|
42
42
|
*
|
|
43
43
|
* If the function throws an exception, the crawler will try to re-crawl the
|
|
44
|
-
* request later, up to the {@
|
|
44
|
+
* request later, up to the {@apilink BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
|
|
45
45
|
* If all the retries fail, the crawler calls the function
|
|
46
|
-
* provided to the {@
|
|
46
|
+
* provided to the {@apilink BasicCrawlerOptions.failedRequestHandler|`failedRequestHandler`} parameter.
|
|
47
47
|
* To make this work, we should **always**
|
|
48
48
|
* let our function throw exceptions rather than catch them.
|
|
49
49
|
* The exceptions are logged to the request using the
|
|
50
|
-
* {@
|
|
50
|
+
* {@apilink Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
|
|
51
51
|
*
|
|
52
52
|
* @deprecated `handleRequestFunction` has been renamed to `requestHandler` and will be removed in a future version.
|
|
53
53
|
* @ignore
|
|
@@ -55,25 +55,25 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
|
|
|
55
55
|
handleRequestFunction?: RequestHandler<Context>;
|
|
56
56
|
/**
|
|
57
57
|
* Static list of URLs to be processed.
|
|
58
|
-
* If not provided, the crawler will open the default request queue when the {@
|
|
59
|
-
* > Alternatively, `requests` parameter of {@
|
|
58
|
+
* If not provided, the crawler will open the default request queue when the {@apilink BasicCrawler.addRequests|`crawler.addRequests()`} function is called.
|
|
59
|
+
* > Alternatively, `requests` parameter of {@apilink BasicCrawler.run|`crawler.run()`} could be used to enqueue the initial requests -
|
|
60
60
|
* it is a shortcut for running `crawler.addRequests()` before the `crawler.run()`.
|
|
61
61
|
*/
|
|
62
62
|
requestList?: RequestList;
|
|
63
63
|
/**
|
|
64
64
|
* Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
|
|
65
|
-
* If not provided, the crawler will open the default request queue when the {@
|
|
66
|
-
* > Alternatively, `requests` parameter of {@
|
|
65
|
+
* If not provided, the crawler will open the default request queue when the {@apilink BasicCrawler.addRequests|`crawler.addRequests()`} function is called.
|
|
66
|
+
* > Alternatively, `requests` parameter of {@apilink BasicCrawler.run|`crawler.run()`} could be used to enqueue the initial requests -
|
|
67
67
|
* it is a shortcut for running `crawler.addRequests()` before the `crawler.run()`.
|
|
68
68
|
*/
|
|
69
69
|
requestQueue?: RequestQueue;
|
|
70
70
|
/**
|
|
71
|
-
* Timeout in which the function passed as {@
|
|
71
|
+
* Timeout in which the function passed as {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`} needs to finish, in seconds.
|
|
72
72
|
* @default 60
|
|
73
73
|
*/
|
|
74
74
|
requestHandlerTimeoutSecs?: number;
|
|
75
75
|
/**
|
|
76
|
-
* Timeout in which the function passed as {@
|
|
76
|
+
* Timeout in which the function passed as {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`} needs to finish, in seconds.
|
|
77
77
|
* @default 60
|
|
78
78
|
* @deprecated `handleRequestTimeoutSecs` has been renamed to `requestHandlerTimeoutSecs` and will be removed in a future version.
|
|
79
79
|
* @ignore
|
|
@@ -81,28 +81,28 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
|
|
|
81
81
|
handleRequestTimeoutSecs?: number;
|
|
82
82
|
/**
|
|
83
83
|
* User-provided function that allows modifying the request object before it gets retried by the crawler.
|
|
84
|
-
* It's executed before each retry for the requests that failed less than {@
|
|
84
|
+
* It's executed before each retry for the requests that failed less than {@apilink BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
|
|
85
85
|
*
|
|
86
|
-
* The function receives the {@
|
|
87
|
-
* where the {@
|
|
86
|
+
* The function receives the {@apilink BasicCrawlingContext} as the first argument,
|
|
87
|
+
* where the {@apilink BasicCrawlingContext.request|`request`} corresponds to the request to be retried.
|
|
88
88
|
* Second argument is the `Error` instance that
|
|
89
89
|
* represents the last error thrown during processing of the request.
|
|
90
90
|
*/
|
|
91
91
|
errorHandler?: ErrorHandler<Context>;
|
|
92
92
|
/**
|
|
93
|
-
* A function to handle requests that failed more than {@
|
|
93
|
+
* A function to handle requests that failed more than {@apilink BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
|
|
94
94
|
*
|
|
95
|
-
* The function receives the {@
|
|
96
|
-
* where the {@
|
|
95
|
+
* The function receives the {@apilink BasicCrawlingContext} as the first argument,
|
|
96
|
+
* where the {@apilink BasicCrawlingContext.request|`request`} corresponds to the failed request.
|
|
97
97
|
* Second argument is the `Error` instance that
|
|
98
98
|
* represents the last error thrown during processing of the request.
|
|
99
99
|
*/
|
|
100
100
|
failedRequestHandler?: ErrorHandler<Context>;
|
|
101
101
|
/**
|
|
102
|
-
* A function to handle requests that failed more than {@
|
|
102
|
+
* A function to handle requests that failed more than {@apilink BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
|
|
103
103
|
*
|
|
104
|
-
* The function receives the {@
|
|
105
|
-
* where the {@
|
|
104
|
+
* The function receives the {@apilink BasicCrawlingContext} as the first argument,
|
|
105
|
+
* where the {@apilink BasicCrawlingContext.request|`request`} corresponds to the failed request.
|
|
106
106
|
* Second argument is the `Error` instance that
|
|
107
107
|
* represents the last error thrown during processing of the request.
|
|
108
108
|
*
|
|
@@ -111,7 +111,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
|
|
|
111
111
|
*/
|
|
112
112
|
handleFailedRequestFunction?: ErrorHandler<Context>;
|
|
113
113
|
/**
|
|
114
|
-
* Indicates how many times the request is retried if {@
|
|
114
|
+
* Indicates how many times the request is retried if {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`} fails.
|
|
115
115
|
* @default 3
|
|
116
116
|
*/
|
|
117
117
|
maxRequestRetries?: number;
|
|
@@ -122,38 +122,44 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
|
|
|
122
122
|
*/
|
|
123
123
|
maxRequestsPerCrawl?: number;
|
|
124
124
|
/**
|
|
125
|
-
* Custom options passed to the underlying {@
|
|
126
|
-
* > *NOTE:* The {@
|
|
127
|
-
* and {@
|
|
125
|
+
* Custom options passed to the underlying {@apilink AutoscaledPool} constructor.
|
|
126
|
+
* > *NOTE:* The {@apilink AutoscaledPoolOptions.runTaskFunction|`runTaskFunction`}
|
|
127
|
+
* and {@apilink AutoscaledPoolOptions.isTaskReadyFunction|`isTaskReadyFunction`} options
|
|
128
128
|
* are provided by the crawler and cannot be overridden.
|
|
129
|
-
* However, we can provide a custom implementation of {@
|
|
129
|
+
* However, we can provide a custom implementation of {@apilink AutoscaledPoolOptions.isFinishedFunction|`isFinishedFunction`}.
|
|
130
130
|
*/
|
|
131
131
|
autoscaledPoolOptions?: AutoscaledPoolOptions;
|
|
132
132
|
/**
|
|
133
133
|
* Sets the minimum concurrency (parallelism) for the crawl. Shortcut for the
|
|
134
|
-
* AutoscaledPool {@
|
|
134
|
+
* AutoscaledPool {@apilink AutoscaledPoolOptions.minConcurrency|`minConcurrency`} option.
|
|
135
135
|
* > *WARNING:* If we set this value too high with respect to the available system memory and CPU, our crawler will run extremely slow or crash.
|
|
136
136
|
* If not sure, it's better to keep the default value and the concurrency will scale up automatically.
|
|
137
137
|
*/
|
|
138
138
|
minConcurrency?: number;
|
|
139
139
|
/**
|
|
140
140
|
* Sets the maximum concurrency (parallelism) for the crawl. Shortcut for the
|
|
141
|
-
* AutoscaledPool {@
|
|
141
|
+
* AutoscaledPool {@apilink AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} option.
|
|
142
142
|
*/
|
|
143
143
|
maxConcurrency?: number;
|
|
144
144
|
/**
|
|
145
145
|
* The maximum number of requests per minute the crawler should run.
|
|
146
146
|
* By default, this is set to `Infinity`, but we can pass any positive, non-zero integer.
|
|
147
|
-
* Shortcut for the AutoscaledPool {@
|
|
147
|
+
* Shortcut for the AutoscaledPool {@apilink AutoscaledPoolOptions.maxTasksPerMinute|`maxTasksPerMinute`} option.
|
|
148
148
|
*/
|
|
149
149
|
maxRequestsPerMinute?: number;
|
|
150
150
|
/**
|
|
151
|
-
*
|
|
152
|
-
*
|
|
151
|
+
* Allows to keep the crawler alive even if the {@apilink RequestQueue} gets empty.
|
|
152
|
+
* By default, the `crawler.run()` will resolve once the queue is empty. With `keepAlive: true` it will keep running,
|
|
153
|
+
* waiting for more requests to come. Use `crawler.teardown()` to exit the crawler.
|
|
154
|
+
*/
|
|
155
|
+
keepAlive?: boolean;
|
|
156
|
+
/**
|
|
157
|
+
* Basic crawler will initialize the {@apilink SessionPool} with the corresponding {@apilink SessionPoolOptions|`sessionPoolOptions`}.
|
|
158
|
+
* The session instance will be than available in the {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`}.
|
|
153
159
|
*/
|
|
154
160
|
useSessionPool?: boolean;
|
|
155
161
|
/**
|
|
156
|
-
* The configuration options for {@
|
|
162
|
+
* The configuration options for {@apilink SessionPool} to use.
|
|
157
163
|
*/
|
|
158
164
|
sessionPoolOptions?: SessionPoolOptions;
|
|
159
165
|
/** @internal */
|
|
@@ -167,29 +173,29 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
|
|
|
167
173
|
* `BasicCrawler` is a low-level tool that requires the user to implement the page
|
|
168
174
|
* download and data extraction functionality themselves.
|
|
169
175
|
* If we want a crawler that already facilitates this functionality,
|
|
170
|
-
* we should consider using {@
|
|
176
|
+
* we should consider using {@apilink CheerioCrawler}, {@apilink PuppeteerCrawler} or {@apilink PlaywrightCrawler}.
|
|
171
177
|
*
|
|
172
|
-
* `BasicCrawler` invokes the user-provided {@
|
|
173
|
-
* for each {@
|
|
174
|
-
* The {@
|
|
175
|
-
* instances provided by the {@
|
|
178
|
+
* `BasicCrawler` invokes the user-provided {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`}
|
|
179
|
+
* for each {@apilink Request} object, which represents a single URL to crawl.
|
|
180
|
+
* The {@apilink Request} objects are fed from the {@apilink RequestList} or {@apilink RequestQueue}
|
|
181
|
+
* instances provided by the {@apilink BasicCrawlerOptions.requestList|`requestList`} or {@apilink BasicCrawlerOptions.requestQueue|`requestQueue`}
|
|
176
182
|
* constructor options, respectively. If neither `requestList` nor `requestQueue` options are provided,
|
|
177
|
-
* the crawler will open the default request queue either when the {@
|
|
178
|
-
* or if `requests` parameter (representing the initial requests) of the {@
|
|
183
|
+
* the crawler will open the default request queue either when the {@apilink BasicCrawler.addRequests|`crawler.addRequests()`} function is called,
|
|
184
|
+
* or if `requests` parameter (representing the initial requests) of the {@apilink BasicCrawler.run|`crawler.run()`} function is provided.
|
|
179
185
|
*
|
|
180
|
-
* If both {@
|
|
181
|
-
* the instance first processes URLs from the {@
|
|
182
|
-
* to the {@
|
|
186
|
+
* If both {@apilink BasicCrawlerOptions.requestList|`requestList`} and {@apilink BasicCrawlerOptions.requestQueue|`requestQueue`} options are used,
|
|
187
|
+
* the instance first processes URLs from the {@apilink RequestList} and automatically enqueues all of them
|
|
188
|
+
* to the {@apilink RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
|
|
183
189
|
*
|
|
184
|
-
* The crawler finishes if there are no more {@
|
|
190
|
+
* The crawler finishes if there are no more {@apilink Request} objects to crawl.
|
|
185
191
|
*
|
|
186
192
|
* New requests are only dispatched when there is enough free CPU and memory available,
|
|
187
|
-
* using the functionality provided by the {@
|
|
188
|
-
* All {@
|
|
193
|
+
* using the functionality provided by the {@apilink AutoscaledPool} class.
|
|
194
|
+
* All {@apilink AutoscaledPool} configuration options can be passed to the {@apilink BasicCrawlerOptions.autoscaledPoolOptions|`autoscaledPoolOptions`}
|
|
189
195
|
* parameter of the `BasicCrawler` constructor.
|
|
190
|
-
* For user convenience, the {@
|
|
191
|
-
* {@
|
|
192
|
-
* underlying {@
|
|
196
|
+
* For user convenience, the {@apilink AutoscaledPoolOptions.minConcurrency|`minConcurrency`} and
|
|
197
|
+
* {@apilink AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} options of the
|
|
198
|
+
* underlying {@apilink AutoscaledPool} constructor are available directly in the `BasicCrawler` constructor.
|
|
193
199
|
*
|
|
194
200
|
* **Example usage:**
|
|
195
201
|
*
|
|
@@ -227,36 +233,36 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawlin
|
|
|
227
233
|
readonly config: Configuration;
|
|
228
234
|
private static readonly CRAWLEE_STATE_KEY;
|
|
229
235
|
/**
|
|
230
|
-
* A reference to the underlying {@
|
|
236
|
+
* A reference to the underlying {@apilink Statistics} class that collects and logs run statistics for requests.
|
|
231
237
|
*/
|
|
232
238
|
readonly stats: Statistics;
|
|
233
239
|
/**
|
|
234
|
-
* A reference to the underlying {@
|
|
240
|
+
* A reference to the underlying {@apilink RequestList} class that manages the crawler's {@apilink Request|requests}.
|
|
235
241
|
* Only available if used by the crawler.
|
|
236
242
|
*/
|
|
237
243
|
requestList?: RequestList;
|
|
238
244
|
/**
|
|
239
245
|
* Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
|
|
240
|
-
* A reference to the underlying {@
|
|
246
|
+
* A reference to the underlying {@apilink RequestQueue} class that manages the crawler's {@apilink Request|requests}.
|
|
241
247
|
* Only available if used by the crawler.
|
|
242
248
|
*/
|
|
243
249
|
requestQueue?: RequestQueue;
|
|
244
250
|
/**
|
|
245
|
-
* A reference to the underlying {@
|
|
251
|
+
* A reference to the underlying {@apilink SessionPool} class that manages the crawler's {@apilink Session|sessions}.
|
|
246
252
|
* Only available if used by the crawler.
|
|
247
253
|
*/
|
|
248
254
|
sessionPool?: SessionPool;
|
|
249
255
|
/**
|
|
250
|
-
* A reference to the underlying {@
|
|
251
|
-
* > *NOTE:* This property is only initialized after calling the {@
|
|
256
|
+
* A reference to the underlying {@apilink AutoscaledPool} class that manages the concurrency of the crawler.
|
|
257
|
+
* > *NOTE:* This property is only initialized after calling the {@apilink BasicCrawler.run|`crawler.run()`} function.
|
|
252
258
|
* We can use it to change the concurrency settings on the fly,
|
|
253
|
-
* to pause the crawler by calling {@
|
|
254
|
-
* or to abort it by calling {@
|
|
259
|
+
* to pause the crawler by calling {@apilink AutoscaledPool.pause|`autoscaledPool.pause()`}
|
|
260
|
+
* or to abort it by calling {@apilink AutoscaledPool.abort|`autoscaledPool.abort()`}.
|
|
255
261
|
*/
|
|
256
262
|
autoscaledPool?: AutoscaledPool;
|
|
257
263
|
/**
|
|
258
|
-
* Default {@
|
|
259
|
-
* See {@
|
|
264
|
+
* Default {@apilink Router} instance that will be used if we don't specify any {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`}.
|
|
265
|
+
* See {@apilink Router.addHandler|`router.addHandler()`} and {@apilink Router.addDefaultHandler|`router.addDefaultHandler()`}.
|
|
260
266
|
*/
|
|
261
267
|
readonly router: RouterHandler<Context>;
|
|
262
268
|
protected log: Log;
|
|
@@ -291,6 +297,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawlin
|
|
|
291
297
|
minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
292
298
|
maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
293
299
|
maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
300
|
+
keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
294
301
|
log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
295
302
|
};
|
|
296
303
|
/**
|
|
@@ -300,7 +307,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawlin
|
|
|
300
307
|
/**
|
|
301
308
|
* Runs the crawler. Returns a promise that gets resolved once all the requests are processed.
|
|
302
309
|
* We can use the `requests` parameter to enqueue the initial requests - it is a shortcut for
|
|
303
|
-
* running {@
|
|
310
|
+
* running {@apilink BasicCrawler.addRequests|`crawler.addRequests()`} before the {@apilink BasicCrawler.run|`crawler.run()`}.
|
|
304
311
|
*
|
|
305
312
|
* @param [requests] The requests to add
|
|
306
313
|
* @param [options] Options for the request queue
|
|
@@ -350,6 +357,14 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawlin
|
|
|
350
357
|
protected _requestFunctionErrorHandler(error: Error, crawlingContext: Context, source: RequestList | RequestQueue): Promise<void>;
|
|
351
358
|
protected _tagUserHandlerError<T>(cb: () => unknown): Promise<T>;
|
|
352
359
|
protected _handleFailedRequestHandler(crawlingContext: Context, error: Error): Promise<void>;
|
|
360
|
+
/**
|
|
361
|
+
* Resolves the most verbose error message from a thrown error
|
|
362
|
+
* @param error The error received
|
|
363
|
+
* @returns The message to be logged
|
|
364
|
+
*/
|
|
365
|
+
protected _getMessageFromError(error: Error, forceStack?: boolean): string | Error | undefined;
|
|
366
|
+
protected _canRequestBeRetried(request: Request, error: Error): boolean;
|
|
367
|
+
protected _augmentContextWithDeprecatedError(context: Context, error: Error): Context;
|
|
353
368
|
/**
|
|
354
369
|
* Updates handledRequestsCount from possibly stored counts,
|
|
355
370
|
* usually after worker migration. Since one of the stores
|
|
@@ -385,8 +400,8 @@ export interface CrawlerAddRequestsResult {
|
|
|
385
400
|
/**
|
|
386
401
|
* A promise which will resolve with the rest of the requests that were added to the queue.
|
|
387
402
|
*
|
|
388
|
-
* Alternatively, we can set {@
|
|
389
|
-
* in the {@
|
|
403
|
+
* Alternatively, we can set {@apilink CrawlerAddRequestsOptions.waitForAllRequestsToBeAdded|`waitForAllRequestsToBeAdded`} to `true`
|
|
404
|
+
* in the {@apilink BasicCrawler.addRequests|`crawler.addRequests()`} options.
|
|
390
405
|
*
|
|
391
406
|
* **Example:**
|
|
392
407
|
*
|
|
@@ -409,9 +424,9 @@ interface HandlePropertyNameChangeData<New, Old> {
|
|
|
409
424
|
allowUndefined?: boolean;
|
|
410
425
|
}
|
|
411
426
|
/**
|
|
412
|
-
* Creates new {@
|
|
413
|
-
* This instance can then serve as a {@
|
|
414
|
-
* Defaults to the {@
|
|
427
|
+
* Creates new {@apilink Router} instance that works based on request labels.
|
|
428
|
+
* This instance can then serve as a {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`} of our {@apilink BasicCrawler}.
|
|
429
|
+
* Defaults to the {@apilink BasicCrawlingContext}.
|
|
415
430
|
*
|
|
416
431
|
* > Serves as a shortcut for using `Router.create<BasicCrawlingContext>()`.
|
|
417
432
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"basic-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/basic-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAItC,OAAO,KAAK,EACR,qBAAqB,EACrB,mBAAmB,EACnB,YAAY,EACZ,eAAe,EACf,SAAS,EACT,OAAO,EACP,WAAW,EACX,cAAc,EACd,4BAA4B,EAC5B,aAAa,EACb,OAAO,EACP,kBAAkB,EACrB,MAAM,eAAe,CAAC;AACvB,OAAO,EAEH,cAAc,EACd,aAAa,EACb,KAAK,eAAe,EAOpB,YAAY,EAEZ,WAAW,EACX,UAAU,
|
|
1
|
+
{"version":3,"file":"basic-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/basic-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAItC,OAAO,KAAK,EACR,qBAAqB,EACrB,mBAAmB,EACnB,YAAY,EACZ,eAAe,EACf,SAAS,EACT,OAAO,EACP,WAAW,EACX,cAAc,EACd,4BAA4B,EAC5B,aAAa,EACb,OAAO,EACP,kBAAkB,EACrB,MAAM,eAAe,CAAC;AACvB,OAAO,EAEH,cAAc,EACd,aAAa,EACb,KAAK,eAAe,EAOpB,YAAY,EAEZ,WAAW,EACX,UAAU,EAIb,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAU,WAAW,EAAE,QAAQ,IAAI,WAAW,EAAE,MAAM,cAAc,CAAC;AAEjF,OAAO,KAAK,EAAE,gBAAgB,EAAE,UAAU,EAAE,SAAS,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAItG,MAAM,WAAW,oBAAoB,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU,CAAE,SAAQ,eAAe,CAAC,QAAQ,CAAC;IAC7G,OAAO,EAAE,YAAY,CAAC;IACtB,YAAY,EAAE,CAAC,OAAO,EAAE,+BAA+B,KAAK,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAC5F,WAAW,EAAE,CAAC,eAAe,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,KAAK,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;CACzF;AAED,gBAAgB;AAChB,MAAM,WAAW,+BAAgC,SAAQ,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC;CAAG;AAarG,oBAAY,cAAc,CAAC,OAAO,SAAS,eAAe,GAAG,oBAAoB,IAAI,CAAC,MAAM,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAE1H,oBAAY,YAAY,CAAC,OAAO,SAAS,eAAe,GAAG,oBAAoB,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAEtI,MAAM,WAAW,mBAAmB,CAAC,OAAO,SAAS,eAAe,GAAG,oBAAoB;IACvF;;;;;;;;;;;;;;;;OAgBG;IACH,cAAc,CAAC,EAAE,cAAc,CAAC,OAAO,CAAC,CAAC;IAEzC;;;;;;;;;;;;;;;;;;;OAmBG;IACH,qBAAqB,CAAC,EAAE,cAAc,CAAC,OAAO,CAAC,CAAC;IAEhD;;;;;OAKG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B;;;;;OAKG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B;;;OAGG;IACH,yBAAyB,CAAC,EAAE,MAAM,CAAC;IAEnC;;;;;OAKG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC;;;;;;;;OAQG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;IAErC;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;IAE7C;;;;;;;;;;OAUG;IACH,2BAA2B,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;IAEpD;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B;;;;;;OAMG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IAE9C;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAE9B;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;OAEG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC,gBAAgB;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC;CACb;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AACH,qBAAa,YAAY,CAAC,OAAO,SAAS,eAAe,GAAG,oBAAoB;IA4FpB,QAAQ,CAAC,MAAM;IA3FvE,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAmB;IAE5D;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC;IAE3B;;;OAGG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B;;;;OAIG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B;;;OAGG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B;;;;;;OAMG;IACH,cAAc,CAAC,EAAE,cAAc,CAAC;IAEhC;;;OAGG;IACH,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,CAA4B;IAEnE,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC;IACnB,SAAS,CAAC,cAAc,EAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACnD,SAAS,CAAC,YAAY,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;IAC/C,SAAS,CAAC,oBAAoB,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;IACvD,SAAS,CAAC,2BAA2B,EAAG,MAAM,CAAC;IAC/C,SAAS,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACxC,SAAS,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACpC,SAAS,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACvC,SAAS,CAAC,kBAAkB,EAAE,kBAAkB,CAAC;IACjD,SAAS,CAAC,cAAc,EAAE,OAAO,CAAC;IAClC,SAAS,CAAC,gBAAgB,uBAA8B;IACxD,SAAS,CAAC,qBAAqB,EAAE,qBAAqB,CAAC;IACvD,SAAS,CAAC,MAAM,EAAE,YAAY,CAAC;IAC/B,OAAO,CAAC,YAAY,CAAC,CAAU;IAE/B,SAAS,CAAC,MAAM,CAAC,YAAY;;;;;;;;;;;;;;;;;;;;MA8B3B;IAEF;;OAEG;gBACS,OAAO,GAAE,mBAAmB,CAAC,OAAO,CAAM,EAAW,MAAM,gBAAkC;IAuKzG;;;;;;;OAOG;IACG,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,MAAM,GAAG,OAAO,GAAG,cAAc,CAAC,EAAE,EAAE,OAAO,CAAC,EAAE,yBAAyB,GAAG,OAAO,CAAC,eAAe,CAAC;IA2CpH,eAAe;IAMf,QAAQ,CAAC,KAAK,SAAS,UAAU,GAAG,UAAU,EAAE,YAAY,QAAc,GAAG,OAAO,CAAC,KAAK,CAAC;IAKjG;;;;OAIG;IACG,WAAW,CAAC,QAAQ,EAAE,CAAC,MAAM,GAAG,OAAO,GAAG,cAAc,CAAC,EAAE,EAAE,OAAO,GAAE,yBAA8B,GAAG,OAAO,CAAC,wBAAwB,CAAC;cAmE9H,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;cAoBtB,kBAAkB,CAAC,eAAe,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAI3E;;OAEG;IACH,SAAS,CAAC,sBAAsB,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM;cAQrD,iBAAiB;IAqCjC;;;OAGG;cACa,iBAAiB;IAmBjC;;;OAGG;cACa,gBAAgB;IAsHhC;;;OAGG;cACa,gBAAgB,CAAC,OAAO,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,MAAM,EAAE,UAAU,SAAI,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAarJ;;OAEG;cACa,oBAAoB;IASpC;;OAEG;cACa,0BAA0B;IAY1C;;OAEG;cACa,4BAA4B,CACxC,KAAK,EAAE,KAAK,EACZ,eAAe,EAAE,OAAO,EACxB,MAAM,EAAE,WAAW,GAAG,YAAY,GACnC,OAAO,CAAC,IAAI,CAAC;cAsCA,oBAAoB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC;cAStD,2BAA2B,CAAC,eAAe,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC;IAelG;;;;OAIG;IACH,SAAS,CAAC,oBAAoB,CAAC,KAAK,EAAE,KAAK,EAAE,UAAU,UAAQ;IAW/D,SAAS,CAAC,oBAAoB,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK;IAe7D,SAAS,CAAC,kCAAkC,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK;IAa3E;;;;;;;OAOG;cACa,wBAAwB,IAAI,OAAO,CAAC,IAAI,CAAC;cAQzC,aAAa,CAAC,QAAQ,SAAS,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,SAAS,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,GAAG,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC;IAQpI;;;OAGG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAc/B,SAAS,CAAC,yBAAyB,CAAC,GAAG,EAAE,GAAG,EAAE,EAC1C,WAAW,EACX,OAAO,EACP,WAAW,EACX,OAAO,EACP,WAAW,EACX,cAAsB,GACzB,EAAE,4BAA4B,CAAC,GAAG,EAAE,GAAG,CAAC;IA0BzC,SAAS,CAAC,2BAA2B,CAAC,OAAO,EAAE,OAAO;CAQzD;AAED,MAAM,WAAW,oBAAoB;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,SAAS,CAAC;CACzB;AAED,MAAM,WAAW,yBAA0B,SAAQ,4BAA4B;IAC3E;;;OAGG;IACH,2BAA2B,CAAC,EAAE,OAAO,CAAC;CACzC;AAED,MAAM,WAAW,wBAAwB;IACrC,aAAa,EAAE,gBAAgB,EAAE,CAAC;IAClC;;;;;;;;;;;;;;;OAeG;IACH,2BAA2B,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;CAC5D;AAED,UAAU,4BAA4B,CAAC,GAAG,EAAE,GAAG;IAC3C,WAAW,CAAC,EAAE,GAAG,CAAC;IAClB,WAAW,CAAC,EAAE,GAAG,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,4BAE5F"}
|