@crawlee/basic 4.0.0-beta.28 → 4.0.0-beta.281
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/index.d.ts +11 -2
- package/index.js +11 -1
- package/internals/autoscaling/autoscaled-pool.d.ts +178 -0
- package/internals/autoscaling/autoscaled-pool.js +389 -0
- package/internals/autoscaling/concurrency-system.d.ts +235 -0
- package/internals/autoscaling/concurrency-system.js +357 -0
- package/internals/autoscaling/cpu-load-signal.d.ts +43 -0
- package/internals/autoscaling/cpu-load-signal.js +46 -0
- package/internals/autoscaling/event-loop-load-signal.d.ts +51 -0
- package/internals/autoscaling/event-loop-load-signal.js +60 -0
- package/internals/autoscaling/index.d.ts +9 -0
- package/internals/autoscaling/index.js +9 -0
- package/internals/autoscaling/load-signal.d.ts +98 -0
- package/internals/autoscaling/load-signal.js +103 -0
- package/internals/autoscaling/memory-load-signal.d.ts +45 -0
- package/internals/autoscaling/memory-load-signal.js +104 -0
- package/internals/autoscaling/snapshotter.d.ts +84 -0
- package/internals/autoscaling/snapshotter.js +67 -0
- package/internals/autoscaling/storage-backend-load-signal.d.ts +56 -0
- package/internals/autoscaling/storage-backend-load-signal.js +73 -0
- package/internals/autoscaling/system-status.d.ts +112 -0
- package/internals/autoscaling/system-status.js +139 -0
- package/internals/autoscaling/weighted-avg.d.ts +5 -0
- package/internals/autoscaling/weighted-avg.js +14 -0
- package/internals/basic-crawler.d.ts +370 -374
- package/internals/basic-crawler.js +1416 -857
- package/internals/cookie-utils.d.ts +40 -0
- package/internals/cookie-utils.js +122 -0
- package/internals/crawler-run.d.ts +76 -0
- package/internals/crawler-run.js +144 -0
- package/internals/crawlers/context-pipeline.d.ts +75 -0
- package/internals/crawlers/context-pipeline.js +109 -0
- package/internals/crawlers/crawler-commons.d.ts +216 -0
- package/internals/crawlers/crawler-commons.js +1 -0
- package/internals/crawlers/error-snapshotter.d.ts +29 -0
- package/internals/crawlers/error-snapshotter.js +116 -0
- package/internals/crawlers/error-tracker.d.ts +51 -0
- package/internals/crawlers/error-tracker.js +316 -0
- package/internals/crawlers/index.d.ts +4 -0
- package/internals/crawlers/index.js +3 -0
- package/internals/crawlers/internals/types.d.ts +7 -0
- package/internals/crawlers/internals/types.js +1 -0
- package/internals/crawlers/statistics.d.ts +326 -0
- package/internals/crawlers/statistics.js +559 -0
- package/internals/crawling-request.d.ts +86 -0
- package/internals/crawling-request.js +127 -0
- package/internals/enqueue-links/enqueue-links.d.ts +147 -0
- package/internals/enqueue-links/enqueue-links.js +35 -0
- package/internals/enqueue-links/index.d.ts +3 -0
- package/internals/enqueue-links/index.js +3 -0
- package/internals/enqueue-links/shared.d.ts +107 -0
- package/internals/enqueue-links/shared.js +228 -0
- package/internals/errors.d.ts +73 -0
- package/internals/errors.js +108 -0
- package/internals/http.d.ts +9 -0
- package/internals/http.js +28 -0
- package/internals/request-timeout.d.ts +54 -0
- package/internals/request-timeout.js +98 -0
- package/internals/router.d.ts +299 -0
- package/internals/router.js +312 -0
- package/internals/send-request.d.ts +4 -4
- package/internals/send-request.js +18 -8
- package/internals/session-pool/consts.d.ts +1 -0
- package/internals/session-pool/consts.js +1 -0
- package/internals/session-pool/errors.d.ts +7 -0
- package/internals/session-pool/errors.js +11 -0
- package/internals/session-pool/fingerprint.d.ts +9 -0
- package/internals/session-pool/fingerprint.js +30 -0
- package/internals/session-pool/index.d.ts +4 -0
- package/internals/session-pool/index.js +4 -0
- package/internals/session-pool/session-pool.d.ts +193 -0
- package/internals/session-pool/session-pool.js +453 -0
- package/internals/session-pool/session.d.ts +151 -0
- package/internals/session-pool/session.js +225 -0
- package/internals/sitemap-request-loader.d.ts +165 -0
- package/internals/sitemap-request-loader.js +492 -0
- package/internals/throttling-request-manager.d.ts +229 -0
- package/internals/throttling-request-manager.js +860 -0
- package/package.json +23 -17
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/internals/basic-crawler.d.ts.map +0 -1
- package/internals/basic-crawler.js.map +0 -1
- package/internals/send-request.d.ts.map +0 -1
- package/internals/send-request.js.map +0 -1
|
@@ -1,25 +1,44 @@
|
|
|
1
|
-
import type { AddRequestsBatchedOptions, AddRequestsBatchedResult,
|
|
2
|
-
import {
|
|
3
|
-
import
|
|
1
|
+
import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, CrawleeLogger, DatasetExportOptions, IProxyConfiguration, IRequestLoader, IRequestManager, RequestsLike, Source, StorageIdentifier, StorageWritePolicy } from '@crawlee/core';
|
|
2
|
+
import { Configuration, Request, Dataset, EventManager, RequestQueue } from '@crawlee/core';
|
|
3
|
+
import { BaseHttpClient } from '@crawlee/http-client';
|
|
4
|
+
import type { Awaitable, Dictionary, ISessionPool, SetStatusMessageOptions, StorageBackend } from '@crawlee/types';
|
|
4
5
|
import { RobotsTxtFile } from '@crawlee/utils';
|
|
5
|
-
import type { ReadonlyDeep
|
|
6
|
-
import
|
|
7
|
-
import {
|
|
6
|
+
import type { ReadonlyDeep } from 'type-fest';
|
|
7
|
+
import { z } from 'zod';
|
|
8
|
+
import type { TaskLoopPredicates } from './autoscaling/autoscaled-pool.js';
|
|
9
|
+
import type { ConcurrencySystemOptions, IConcurrencySystem } from './autoscaling/concurrency-system.js';
|
|
10
|
+
import { ConcurrencySystem } from './autoscaling/concurrency-system.js';
|
|
11
|
+
import type { FinalStatistics } from './autoscaling/system-status.js';
|
|
12
|
+
import type { ContextMiddleware } from './crawlers/context-pipeline.js';
|
|
13
|
+
import { ContextPipeline } from './crawlers/context-pipeline.js';
|
|
14
|
+
import type { CrawlingContext, TypedRequestsLike } from './crawlers/crawler-commons.js';
|
|
15
|
+
import type { IStatistics, StatisticState } from './crawlers/statistics.js';
|
|
16
|
+
import type { EnqueueUrlsOptions, SkippedRequestCallback } from './enqueue-links/index.js';
|
|
17
|
+
import type { GetUserDataFromRequest, RouterHandler, RouterRoutes } from './router.js';
|
|
8
18
|
export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
|
|
9
19
|
}
|
|
20
|
+
export { navigationDeadlineKey, remainingNavigationWindowMillis } from './request-timeout.js';
|
|
10
21
|
export type RequestHandler<Context extends CrawlingContext = CrawlingContext> = (inputs: Context) => Awaitable<void>;
|
|
11
|
-
|
|
12
|
-
|
|
22
|
+
/**
|
|
23
|
+
* An error handler receives the crawling context and the error that was thrown while processing the request.
|
|
24
|
+
*
|
|
25
|
+
* Unlike the {@link RequestHandler}, an error handler may run before the context pipeline has finished
|
|
26
|
+
* building the full context (e.g. when navigation or session setup fails). Therefore only `BaseContext` is
|
|
27
|
+
* guaranteed to be present, while the extra properties added by the pipeline and `extendContext` (the
|
|
28
|
+
* difference between `BaseContext` and `ExtendedContext`) are only available as a `Partial`.
|
|
29
|
+
*/
|
|
30
|
+
export type ErrorHandler<BaseContext extends CrawlingContext = CrawlingContext, ExtendedContext extends BaseContext = BaseContext> = (inputs: BaseContext & Partial<ExtendedContext>, error: Error) => Awaitable<void>;
|
|
31
|
+
export interface StatusMessageCallbackParams<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any, any, any, any> = BasicCrawler<Context>> {
|
|
13
32
|
state: StatisticState;
|
|
14
33
|
crawler: Crawler;
|
|
15
34
|
previousState: StatisticState;
|
|
16
35
|
message: string;
|
|
17
36
|
}
|
|
18
|
-
export type StatusMessageCallback<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any> = BasicCrawler<Context>> = (params: StatusMessageCallbackParams<Context, Crawler>) => Awaitable<void>;
|
|
37
|
+
export type StatusMessageCallback<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any, any, any, any> = BasicCrawler<Context>> = (params: StatusMessageCallbackParams<Context, Crawler>) => Awaitable<void>;
|
|
19
38
|
export type RequireContextPipeline<DefaultContextType extends CrawlingContext, FinalContextType extends DefaultContextType> = DefaultContextType extends FinalContextType ? {} : {
|
|
20
39
|
contextPipelineBuilder: () => ContextPipeline<CrawlingContext, FinalContextType>;
|
|
21
40
|
};
|
|
22
|
-
export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> {
|
|
41
|
+
export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> {
|
|
23
42
|
/**
|
|
24
43
|
* User-provided function that performs the logic of the crawler. It is called for each URL to crawl.
|
|
25
44
|
*
|
|
@@ -35,11 +54,21 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
|
|
|
35
54
|
* To make this work, we should **always**
|
|
36
55
|
* let our function throw exceptions rather than catch them.
|
|
37
56
|
* The exceptions are logged to the request using the
|
|
38
|
-
* {@link
|
|
57
|
+
* {@link CrawlingRequest.pushErrorMessage|`request.pushErrorMessage()`} function.
|
|
39
58
|
*/
|
|
40
|
-
requestHandler?: RequestHandler<ExtendedContext>;
|
|
59
|
+
requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
|
|
41
60
|
/**
|
|
42
|
-
* Allows the user to extend the crawling context
|
|
61
|
+
* Allows the user to extend the crawling context with custom functionality (helpers, references, etc.).
|
|
62
|
+
*
|
|
63
|
+
* `extendContext` runs *before* navigation, so the returned members are visible to the
|
|
64
|
+
* `preNavigationHooks`, `postNavigationHooks`, and the `requestHandler` alike. As a consequence,
|
|
65
|
+
* the `context` passed to `extendContext` is the pre-navigation {@link CrawlingContext} and does
|
|
66
|
+
* **not** include navigation-dependent members (e.g. `page`, `response`, `$`, `body`). If you need
|
|
67
|
+
* those, use a `postNavigationHook` or the `requestHandler` instead.
|
|
68
|
+
*
|
|
69
|
+
* The second argument is the same {@link CleanupRegistrar|`onCleanup`} registrar used by
|
|
70
|
+
* {@link ContextPipeline.compose}. Use it to register teardown callbacks that run after the
|
|
71
|
+
* request handler or a pipeline failure, in reverse registration order.
|
|
43
72
|
*
|
|
44
73
|
* **Example usage:**
|
|
45
74
|
*
|
|
@@ -59,7 +88,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
|
|
|
59
88
|
* });
|
|
60
89
|
* ```
|
|
61
90
|
*/
|
|
62
|
-
extendContext?:
|
|
91
|
+
extendContext?: ContextMiddleware<CrawlingContext, ContextExtension>;
|
|
63
92
|
/**
|
|
64
93
|
* *Intended for BasicCrawler subclasses*. Prepares a context pipeline that transforms the initial crawling context into the shape given by the `Context` type parameter.
|
|
65
94
|
*
|
|
@@ -68,23 +97,25 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
|
|
|
68
97
|
contextPipelineBuilder?: () => ContextPipeline<CrawlingContext, Context>;
|
|
69
98
|
/**
|
|
70
99
|
* Static list of URLs to be processed.
|
|
71
|
-
*
|
|
72
|
-
*
|
|
73
|
-
*
|
|
100
|
+
*
|
|
101
|
+
* @deprecated Use the `requestManager` option instead. To combine a read-only loader (such as a `RequestList`)
|
|
102
|
+
* with a writable queue, build a tandem with {@link IRequestLoader.toTandem|`requestList.toTandem(requestQueue)`}
|
|
103
|
+
* and pass the result as `requestManager`. When both `requestList` and `requestQueue` are provided, they are
|
|
104
|
+
* combined into a tandem automatically.
|
|
74
105
|
*/
|
|
75
|
-
requestList?:
|
|
106
|
+
requestList?: IRequestLoader;
|
|
76
107
|
/**
|
|
77
108
|
* Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
|
|
78
|
-
*
|
|
79
|
-
*
|
|
80
|
-
* it
|
|
109
|
+
*
|
|
110
|
+
* @deprecated Use the `requestManager` option instead. A `RequestQueue` is itself a request manager, so you can
|
|
111
|
+
* pass it directly as `requestManager`.
|
|
81
112
|
*/
|
|
82
|
-
requestQueue?:
|
|
113
|
+
requestQueue?: RequestQueue;
|
|
83
114
|
/**
|
|
84
|
-
*
|
|
115
|
+
* Manager of requests that should be processed by the crawler. Mutually exclusive with the deprecated
|
|
116
|
+
* `requestQueue` and `requestList` options.
|
|
85
117
|
*
|
|
86
|
-
*
|
|
87
|
-
* If using this, the type of `BasicCrawler.requestQueue` may not be fully compatible with the `RequestProvider` class.
|
|
118
|
+
* If not provided, the crawler will open the default {@link RequestQueue} when it is first needed.
|
|
88
119
|
*/
|
|
89
120
|
requestManager?: IRequestManager;
|
|
90
121
|
/**
|
|
@@ -113,27 +144,21 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
|
|
|
113
144
|
failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
|
|
114
145
|
/**
|
|
115
146
|
* Specifies the maximum number of retries allowed for a request if its processing fails.
|
|
116
|
-
* This includes retries due to navigation errors or errors thrown from user-supplied
|
|
117
|
-
* (`requestHandler`, `preNavigationHooks`, `postNavigationHooks`).
|
|
118
|
-
*
|
|
119
|
-
* This limit does not apply to retries triggered by session rotation
|
|
120
|
-
* (see {@link BasicCrawlerOptions.maxSessionRotations|`maxSessionRotations`}).
|
|
147
|
+
* This includes retries due to navigation errors, session/proxy errors, or errors thrown from user-supplied
|
|
148
|
+
* functions (`requestHandler`, `preNavigationHooks`, `postNavigationHooks`).
|
|
121
149
|
* @default 3
|
|
122
150
|
*/
|
|
123
151
|
maxRequestRetries?: number;
|
|
124
152
|
/**
|
|
125
|
-
* Indicates how much time (in seconds) to wait before crawling another same domain request.
|
|
153
|
+
* Indicates how much time (in seconds) to wait before crawling another same domain request. Subdomains are
|
|
154
|
+
* paced together with the site they belong to.
|
|
155
|
+
*
|
|
156
|
+
* Offered to the crawler's request manager as a `minIntervalEverywhere` {@link PacingSignal}; a manager that
|
|
157
|
+
* already paces every domain it dispatches to takes it, so no domain ends up with two clocks. Otherwise the
|
|
158
|
+
* crawler wraps its request manager in a {@link ThrottlingRequestManager} of its own.
|
|
126
159
|
* @default 0
|
|
127
160
|
*/
|
|
128
161
|
sameDomainDelaySecs?: number;
|
|
129
|
-
/**
|
|
130
|
-
* Maximum number of session rotations per request.
|
|
131
|
-
* The crawler will automatically rotate the session in case of a proxy error or if it gets blocked by the website.
|
|
132
|
-
*
|
|
133
|
-
* The session rotations are not counted towards the {@link BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} limit.
|
|
134
|
-
* @default 10
|
|
135
|
-
*/
|
|
136
|
-
maxSessionRotations?: number;
|
|
137
162
|
/**
|
|
138
163
|
* Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.
|
|
139
164
|
* This value should always be set in order to prevent infinite loops in misconfigured crawlers.
|
|
@@ -147,46 +172,82 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
|
|
|
147
172
|
*/
|
|
148
173
|
maxCrawlDepth?: number;
|
|
149
174
|
/**
|
|
150
|
-
*
|
|
151
|
-
*
|
|
152
|
-
*
|
|
153
|
-
*
|
|
154
|
-
*
|
|
175
|
+
* Lets you override the predicates that steer the crawler's task loop: `isTaskReadyFunction` (may another request
|
|
176
|
+
* start?) and `isFinishedFunction` (is the crawl over?). The task itself — fetching a request and running it
|
|
177
|
+
* through the pipeline — is owned by the crawler and cannot be overridden.
|
|
178
|
+
*
|
|
179
|
+
* Concurrency is configured elsewhere — through the `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute`
|
|
180
|
+
* shortcuts, or a {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} for finer control.
|
|
181
|
+
*/
|
|
182
|
+
taskLoopOptions?: TaskLoopPredicates;
|
|
183
|
+
/**
|
|
184
|
+
* A pre-configured concurrency governor — the component that decides whether there is free compute for one more
|
|
185
|
+
* task. Typically a {@link ConcurrencySystem}, though any {@link IConcurrencySystem} is accepted. All
|
|
186
|
+
* scaling configuration (min/max/desired concurrency, scaling ratios, `maxTasksPerMinute`, snapshotter tuning)
|
|
187
|
+
* lives on the instance itself.
|
|
188
|
+
*
|
|
189
|
+
* Inject the *same* instance into several concurrent crawlers to cap their **combined** concurrency against a
|
|
190
|
+
* single budget. Each crawler still builds and drives its own {@link AutoscaledPool}; only the load/scaling
|
|
191
|
+
* accounting is shared.
|
|
192
|
+
*
|
|
193
|
+
* Mutually exclusive with the `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute`
|
|
194
|
+
* shortcuts, which configure the default system this one replaces — combining the two throws.
|
|
195
|
+
*
|
|
196
|
+
* You own a supplied system's lifecycle: `start()` it before `run()` (which throws otherwise) and `stop()` it once
|
|
197
|
+
* every crawler borrowing it has finished. The crawler does neither on your behalf.
|
|
155
198
|
*/
|
|
156
|
-
|
|
199
|
+
concurrencySystem?: IConcurrencySystem;
|
|
157
200
|
/**
|
|
158
201
|
* Sets the minimum concurrency (parallelism) for the crawl. Shortcut for the
|
|
159
|
-
*
|
|
202
|
+
* {@link ConcurrencySystemOptions.minConcurrency|`minConcurrency`} option of the crawler's default
|
|
203
|
+
* {@link ConcurrencySystem}.
|
|
160
204
|
* > *WARNING:* If we set this value too high with respect to the available system memory and CPU, our crawler will run extremely slow or crash.
|
|
161
205
|
* If not sure, it's better to keep the default value and the concurrency will scale up automatically.
|
|
162
206
|
*/
|
|
163
207
|
minConcurrency?: number;
|
|
164
208
|
/**
|
|
165
209
|
* Sets the maximum concurrency (parallelism) for the crawl. Shortcut for the
|
|
166
|
-
*
|
|
210
|
+
* {@link ConcurrencySystemOptions.maxConcurrency|`maxConcurrency`} option of the crawler's default
|
|
211
|
+
* {@link ConcurrencySystem}.
|
|
167
212
|
*/
|
|
168
213
|
maxConcurrency?: number;
|
|
214
|
+
/**
|
|
215
|
+
* Sets the concurrency (parallelism) the crawl starts with, before any scaling happens. Shortcut for the
|
|
216
|
+
* {@link ConcurrencySystemOptions.desiredConcurrency|`desiredConcurrency`} option of the crawler's default
|
|
217
|
+
* {@link ConcurrencySystem}. Defaults to `minConcurrency`.
|
|
218
|
+
*/
|
|
219
|
+
initialConcurrency?: number;
|
|
169
220
|
/**
|
|
170
221
|
* The maximum number of requests per minute the crawler should run.
|
|
171
222
|
* By default, this is set to `Infinity`, but we can pass any positive, non-zero integer.
|
|
172
|
-
* Shortcut for the
|
|
223
|
+
* Shortcut for the {@link ConcurrencySystemOptions.maxTasksPerMinute|`maxTasksPerMinute`} option of the
|
|
224
|
+
* crawler's default {@link ConcurrencySystem}.
|
|
173
225
|
*/
|
|
174
226
|
maxRequestsPerMinute?: number;
|
|
175
227
|
/**
|
|
176
228
|
* Allows to keep the crawler alive even if the {@link RequestQueue} gets empty.
|
|
177
229
|
* By default, the `crawler.run()` will resolve once the queue is empty. With `keepAlive: true` it will keep running,
|
|
178
|
-
* waiting for more requests to come.
|
|
230
|
+
* waiting for more requests to come.
|
|
231
|
+
*
|
|
232
|
+
* `crawler.stop()` only stops the crawler from dispatching new requests — it does not wait for or drain
|
|
233
|
+
* requests already sitting in the queue. If this crawler is fed by another crawler or an external producer,
|
|
234
|
+
* calling `stop()` as soon as the producer finishes can abandon requests the producer already enqueued but
|
|
235
|
+
* this crawler had not yet picked up.
|
|
236
|
+
*
|
|
237
|
+
* To shut down only once the queue is genuinely empty and the producer has stopped, override
|
|
238
|
+
* {@link BasicCrawlerOptions.taskLoopOptions|`taskLoopOptions.isFinishedFunction`} to check both
|
|
239
|
+
* `(await requestManager.checkReadiness()).status === 'finished'` and the producer's own running/finished flag.
|
|
179
240
|
*/
|
|
180
241
|
keepAlive?: boolean;
|
|
181
242
|
/**
|
|
182
|
-
*
|
|
183
|
-
*
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
*
|
|
243
|
+
* An existing session pool instance to use. When provided, the crawler will use this pool directly instead of
|
|
244
|
+
* creating a new one, enabling session sharing across multiple crawlers. The crawler will not tear down a shared
|
|
245
|
+
* pool — the caller is responsible for its lifecycle.
|
|
246
|
+
*
|
|
247
|
+
* Accepts the built-in {@link SessionPool} or any object implementing the {@link ISessionPool} interface,
|
|
248
|
+
* so custom session-management strategies can be plugged in.
|
|
188
249
|
*/
|
|
189
|
-
|
|
250
|
+
sessionPool?: ISessionPool;
|
|
190
251
|
/**
|
|
191
252
|
* Defines the length of the interval for calling the `setStatusMessage` in seconds.
|
|
192
253
|
*/
|
|
@@ -208,12 +269,23 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
|
|
|
208
269
|
* ```
|
|
209
270
|
*/
|
|
210
271
|
statusMessageCallback?: StatusMessageCallback;
|
|
272
|
+
/**
|
|
273
|
+
* HTTP status codes that indicate the session should be retired.
|
|
274
|
+
*
|
|
275
|
+
* A 429 from a domain covered by a {@link ThrottlingRequestManager} is handled as a rate limit before
|
|
276
|
+
* this is consulted, so removing 429 here only affects domains that manager does not cover.
|
|
277
|
+
*
|
|
278
|
+
* @default [401, 403, 429]
|
|
279
|
+
*/
|
|
280
|
+
blockedStatusCodes?: number[];
|
|
211
281
|
/**
|
|
212
282
|
* If set to `true`, the crawler will automatically try to bypass any detected bot protection.
|
|
213
283
|
*
|
|
214
284
|
* Currently supports:
|
|
215
285
|
* - [**Cloudflare** Bot Management](https://www.cloudflare.com/products/bot-management/)
|
|
286
|
+
* - [**AWS WAF** Challenge and CAPTCHA](https://docs.aws.amazon.com/waf/latest/developerguide/waf-captcha-and-challenge.html)
|
|
216
287
|
* - [**Google Search** Rate Limiting](https://www.google.com/sorry/)
|
|
288
|
+
* - **Imperva Incapsula**
|
|
217
289
|
*/
|
|
218
290
|
retryOnBlocked?: boolean;
|
|
219
291
|
/**
|
|
@@ -232,46 +304,64 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
|
|
|
232
304
|
* 1. based on robots.txt file,
|
|
233
305
|
* 2. because they don't match enqueueLinks filters,
|
|
234
306
|
* 3. because they are redirected to a URL that doesn't match the enqueueLinks strategy,
|
|
235
|
-
* 4.
|
|
307
|
+
* 4. because the {@link BasicCrawlerOptions.maxRequestsPerCrawl|`maxRequestsPerCrawl`} limit has been reached,
|
|
308
|
+
* 5. or by {@link RestrictedCrawlingContext.skipRequest|`context.skipRequest()`}.
|
|
236
309
|
*/
|
|
237
310
|
onSkippedRequest?: SkippedRequestCallback;
|
|
238
|
-
/** @internal */
|
|
239
|
-
log?: Log;
|
|
240
|
-
/**
|
|
241
|
-
* Enables experimental features of Crawlee, which can alter the behavior of the crawler.
|
|
242
|
-
* WARNING: these options are not guaranteed to be stable and may change or be removed at any time.
|
|
243
|
-
*/
|
|
244
|
-
experiments?: CrawlerExperiments;
|
|
245
311
|
/**
|
|
246
|
-
*
|
|
247
|
-
*
|
|
312
|
+
* A preconfigured statistics instance. When provided, the crawler records into it instead of building its own and
|
|
313
|
+
* will not `reset()` it between `run()` calls. Accepts the built-in {@link Statistics} or any object
|
|
314
|
+
* implementing {@link IStatistics}.
|
|
315
|
+
*
|
|
316
|
+
* Custom fields declared via {@link StatisticsOptions.stateExtension|`stateExtension`} are carried over to
|
|
317
|
+
* {@link BasicCrawler.statistics|`crawler.statistics.state`}:
|
|
318
|
+
*
|
|
319
|
+
* ```ts
|
|
320
|
+
* const statistics = new Statistics({ stateExtension: { defaultState: { productsFound: 0 } } });
|
|
321
|
+
*
|
|
322
|
+
* const crawler = new BasicCrawler({
|
|
323
|
+
* statistics,
|
|
324
|
+
* requestHandler: async () => {
|
|
325
|
+
* statistics.state.productsFound++;
|
|
326
|
+
* },
|
|
327
|
+
* });
|
|
328
|
+
*
|
|
329
|
+
* await crawler.run();
|
|
330
|
+
* // the custom fields are typed on `crawler.statistics` too
|
|
331
|
+
* console.log(crawler.statistics.state.productsFound);
|
|
332
|
+
* ```
|
|
248
333
|
*/
|
|
249
|
-
|
|
334
|
+
statistics?: IStatistics<StatisticStateExtension>;
|
|
250
335
|
/**
|
|
251
336
|
* HTTP client implementation for the `sendRequest` context helper and for plain HTTP crawling.
|
|
252
|
-
* Defaults to
|
|
337
|
+
* Defaults to {@link ImpitHttpClient} when `@crawlee/impit-client` is installed, otherwise {@link FetchHttpClient}.
|
|
253
338
|
*/
|
|
254
339
|
httpClient?: BaseHttpClient;
|
|
255
340
|
/**
|
|
256
341
|
* If set, the crawler will be configured for all connections to use
|
|
257
342
|
* the Proxy URLs provided and rotated according to the configuration.
|
|
258
343
|
*/
|
|
259
|
-
proxyConfiguration?:
|
|
344
|
+
proxyConfiguration?: IProxyConfiguration;
|
|
260
345
|
/**
|
|
261
346
|
* Custom configuration to use for this crawler.
|
|
262
347
|
* If provided, the crawler will use its own ServiceLocator instance instead of the global one.
|
|
263
348
|
*/
|
|
264
349
|
configuration?: Configuration;
|
|
265
350
|
/**
|
|
266
|
-
* Custom storage
|
|
351
|
+
* Custom storage backend to use for this crawler.
|
|
267
352
|
* If provided, the crawler will use its own ServiceLocator instance instead of the global one.
|
|
268
353
|
*/
|
|
269
|
-
|
|
354
|
+
storageBackend?: StorageBackend;
|
|
270
355
|
/**
|
|
271
356
|
* Custom event manager to use for this crawler.
|
|
272
357
|
* If provided, the crawler will use its own ServiceLocator instance instead of the global one.
|
|
273
358
|
*/
|
|
274
359
|
eventManager?: EventManager;
|
|
360
|
+
/**
|
|
361
|
+
* Custom logger to use for this crawler.
|
|
362
|
+
* If provided, the crawler will use its own ServiceLocator instance instead of the global one.
|
|
363
|
+
*/
|
|
364
|
+
logger?: CrawleeLogger;
|
|
275
365
|
/**
|
|
276
366
|
* A unique identifier for the crawler instance. This ID is used to isolate the state returned by
|
|
277
367
|
* {@link BasicCrawler.useState|`crawler.useState()`} from other crawler instances.
|
|
@@ -284,246 +374,153 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
|
|
|
284
374
|
*
|
|
285
375
|
*/
|
|
286
376
|
id?: string;
|
|
287
|
-
}
|
|
288
|
-
/**
|
|
289
|
-
* A set of options that you can toggle to enable experimental features in Crawlee.
|
|
290
|
-
*
|
|
291
|
-
* NOTE: These options will not respect semantic versioning and may be removed or changed at any time. Use at your own risk.
|
|
292
|
-
* If you do use these and encounter issues, please report them to us.
|
|
293
|
-
*/
|
|
294
|
-
export interface CrawlerExperiments {
|
|
295
377
|
/**
|
|
296
|
-
*
|
|
297
|
-
*
|
|
298
|
-
*
|
|
299
|
-
*
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
}
|
|
303
|
-
|
|
304
|
-
*
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
*
|
|
308
|
-
* `BasicCrawler` is a low-level tool that requires the user to implement the page
|
|
309
|
-
* download and data extraction functionality themselves.
|
|
310
|
-
* If we want a crawler that already facilitates this functionality,
|
|
311
|
-
* we should consider using {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler}.
|
|
312
|
-
*
|
|
313
|
-
* `BasicCrawler` invokes the user-provided {@link BasicCrawlerOptions.requestHandler|`requestHandler`}
|
|
314
|
-
* for each {@link Request} object, which represents a single URL to crawl.
|
|
315
|
-
* The {@link Request} objects are fed from the {@link RequestList} or {@link RequestQueue}
|
|
316
|
-
* instances provided by the {@link BasicCrawlerOptions.requestList|`requestList`} or {@link BasicCrawlerOptions.requestQueue|`requestQueue`}
|
|
317
|
-
* constructor options, respectively. If neither `requestList` nor `requestQueue` options are provided,
|
|
318
|
-
* the crawler will open the default request queue either when the {@link BasicCrawler.addRequests|`crawler.addRequests()`} function is called,
|
|
319
|
-
* or if `requests` parameter (representing the initial requests) of the {@link BasicCrawler.run|`crawler.run()`} function is provided.
|
|
320
|
-
*
|
|
321
|
-
* If both {@link BasicCrawlerOptions.requestList|`requestList`} and {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are used,
|
|
322
|
-
* the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
|
|
323
|
-
* to the {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
|
|
324
|
-
*
|
|
325
|
-
* The crawler finishes if there are no more {@link Request} objects to crawl.
|
|
326
|
-
*
|
|
327
|
-
* New requests are only dispatched when there is enough free CPU and memory available,
|
|
328
|
-
* using the functionality provided by the {@link AutoscaledPool} class.
|
|
329
|
-
* All {@link AutoscaledPool} configuration options can be passed to the {@link BasicCrawlerOptions.autoscaledPoolOptions|`autoscaledPoolOptions`}
|
|
330
|
-
* parameter of the `BasicCrawler` constructor.
|
|
331
|
-
* For user convenience, the {@link AutoscaledPoolOptions.minConcurrency|`minConcurrency`} and
|
|
332
|
-
* {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} options of the
|
|
333
|
-
* underlying {@link AutoscaledPool} constructor are available directly in the `BasicCrawler` constructor.
|
|
334
|
-
*
|
|
335
|
-
* **Example usage:**
|
|
336
|
-
*
|
|
337
|
-
* ```javascript
|
|
338
|
-
* import { BasicCrawler, Dataset } from 'crawlee';
|
|
339
|
-
*
|
|
340
|
-
* // Create a crawler instance
|
|
341
|
-
* const crawler = new BasicCrawler({
|
|
342
|
-
* async requestHandler({ request, sendRequest }) {
|
|
343
|
-
* // 'request' contains an instance of the Request class
|
|
344
|
-
* // Here we simply fetch the HTML of the page and store it to a dataset
|
|
345
|
-
* const { body } = await sendRequest({
|
|
346
|
-
* url: request.url,
|
|
347
|
-
* method: request.method,
|
|
348
|
-
* body: request.payload,
|
|
349
|
-
* headers: request.headers,
|
|
350
|
-
* });
|
|
351
|
-
*
|
|
352
|
-
* await Dataset.pushData({
|
|
353
|
-
* url: request.url,
|
|
354
|
-
* html: body,
|
|
355
|
-
* })
|
|
356
|
-
* },
|
|
357
|
-
* });
|
|
358
|
-
*
|
|
359
|
-
* // Enqueue the initial requests and run the crawler
|
|
360
|
-
* await crawler.run([
|
|
361
|
-
* 'http://www.example.com/page-1',
|
|
362
|
-
* 'http://www.example.com/page-2',
|
|
363
|
-
* ]);
|
|
364
|
-
* ```
|
|
365
|
-
* @category Crawlers
|
|
366
|
-
*/
|
|
367
|
-
export declare class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> {
|
|
368
|
-
protected static readonly CRAWLEE_STATE_KEY = "CRAWLEE_STATE";
|
|
369
|
-
/**
|
|
370
|
-
* Tracks crawler instances that accessed shared state without having an explicit id.
|
|
371
|
-
* Used to detect and warn about multiple crawlers sharing the same state.
|
|
378
|
+
* Makes the storage writes performed while handling a request atomic with respect to the request
|
|
379
|
+
* succeeding: they are recorded in a {@link StorageTransaction} spanning the whole request
|
|
380
|
+
* lifecycle and only applied when the request handler succeeds, so a thrown handler leaves no partial
|
|
381
|
+
* writes behind and a retry does not double-write. Reads within the handler see its own writes.
|
|
382
|
+
*
|
|
383
|
+
* `false` disables the mechanism entirely; an object overrides the per-storage-type
|
|
384
|
+
* {@link StorageWritePolicy} (e.g. `{ requestQueue: 'deferred' }` for all-or-nothing enqueues).
|
|
385
|
+
* {@link withDirectStorageAccess} is the per-call-site escape hatch; `useState()` is deliberately
|
|
386
|
+
* *not* transactional.
|
|
387
|
+
*
|
|
388
|
+
* @default true
|
|
372
389
|
*/
|
|
373
|
-
|
|
390
|
+
transactionalStorage?: boolean | Partial<StorageWritePolicy>;
|
|
374
391
|
/**
|
|
375
|
-
*
|
|
392
|
+
* An array of HTTP response [Status Codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) to be excluded from error consideration.
|
|
393
|
+
* By default, status codes >= 500 trigger errors.
|
|
376
394
|
*/
|
|
377
|
-
|
|
395
|
+
ignoreHttpErrorStatusCodes?: number[];
|
|
378
396
|
/**
|
|
379
|
-
*
|
|
380
|
-
*
|
|
397
|
+
* An array of additional HTTP response [Status Codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) to be treated as errors.
|
|
398
|
+
* By default, status codes >= 500 trigger errors.
|
|
381
399
|
*/
|
|
382
|
-
|
|
400
|
+
additionalHttpErrorStatusCodes?: number[];
|
|
401
|
+
}
|
|
402
|
+
export declare class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> {
|
|
403
|
+
#private;
|
|
383
404
|
/**
|
|
384
|
-
*
|
|
385
|
-
*
|
|
386
|
-
* Only available if used by the crawler.
|
|
405
|
+
* The statistics instance collecting the crawler's run statistics - either the injected `statistics` option or a
|
|
406
|
+
* crawler-built default. Typed as {@link IStatistics} so custom implementations can be plugged in.
|
|
387
407
|
*/
|
|
388
|
-
|
|
408
|
+
get statistics(): IStatistics<StatisticStateExtension>;
|
|
389
409
|
/**
|
|
390
|
-
* The main request-handling component of the crawler. It
|
|
410
|
+
* The main request-handling component of the crawler. It manages the requests that the crawler processes,
|
|
411
|
+
* combining any provided request loader and/or queue. It's initialized during the crawler startup or lazily
|
|
412
|
+
* via {@link BasicCrawler.getRequestManager|`getRequestManager()`}.
|
|
391
413
|
*/
|
|
392
|
-
protected requestManager
|
|
414
|
+
protected get requestManager(): IRequestManager | undefined;
|
|
393
415
|
/**
|
|
394
|
-
* A reference to the underlying
|
|
395
|
-
*
|
|
416
|
+
* A reference to the underlying session pool that manages the crawler's {@link Session|sessions}. Typed as
|
|
417
|
+
* {@link ISessionPool} so custom implementations can be plugged in via the `sessionPool` constructor option.
|
|
396
418
|
*/
|
|
397
|
-
sessionPool
|
|
419
|
+
get sessionPool(): ISessionPool;
|
|
420
|
+
/** Whether a {@link BasicCrawler.run|`run()`} is in progress on this instance. */
|
|
421
|
+
get running(): boolean;
|
|
398
422
|
/**
|
|
399
|
-
*
|
|
400
|
-
*
|
|
401
|
-
*
|
|
402
|
-
*
|
|
403
|
-
*
|
|
423
|
+
* The concurrency governor the run in progress is booking its requests against — either the
|
|
424
|
+
* {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} that was injected, or the default the
|
|
425
|
+
* crawler built for itself. Read it for telemetry: `desiredConcurrency`, `currentConcurrency`, `isRunning`.
|
|
426
|
+
*
|
|
427
|
+
* > *NOTE:* `undefined` outside a run, and a crawler-owned default is rebuilt for every run — so read it
|
|
428
|
+
* during a run rather than caching it across runs.
|
|
429
|
+
*
|
|
430
|
+
* {@link IConcurrencySystem} is deliberately read-only. Tuning concurrency *while a crawl is running* means
|
|
431
|
+
* owning the instance: build a {@link ConcurrencySystem} yourself and inject it, then set
|
|
432
|
+
* `minConcurrency`/`maxConcurrency`/`desiredConcurrency` on your own reference.
|
|
404
433
|
*/
|
|
405
|
-
|
|
434
|
+
get concurrencySystem(): IConcurrencySystem | undefined;
|
|
406
435
|
/**
|
|
407
|
-
* A reference to the underlying {@link
|
|
436
|
+
* A reference to the underlying {@link IProxyConfiguration} instance that manages the crawler's proxies.
|
|
408
437
|
* Only available if used by the crawler.
|
|
409
438
|
*/
|
|
410
|
-
proxyConfiguration?:
|
|
439
|
+
readonly proxyConfiguration?: IProxyConfiguration;
|
|
411
440
|
/**
|
|
412
441
|
* Default {@link Router} instance that will be used if we don't specify any {@link BasicCrawlerOptions.requestHandler|`requestHandler`}.
|
|
413
442
|
* See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
|
|
414
443
|
*/
|
|
415
|
-
readonly router: RouterHandler<Context>;
|
|
416
|
-
|
|
417
|
-
private _contextPipeline?;
|
|
444
|
+
readonly router: RouterHandler<Context, Routes>;
|
|
445
|
+
/** @internal */
|
|
418
446
|
get contextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext>;
|
|
419
|
-
|
|
420
|
-
hasFinishedBefore: boolean;
|
|
421
|
-
|
|
422
|
-
protected requestHandler: RequestHandler<ExtendedContext>;
|
|
423
|
-
protected
|
|
424
|
-
protected
|
|
425
|
-
protected
|
|
426
|
-
protected
|
|
427
|
-
protected
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
protected maxSessionRotations: number;
|
|
432
|
-
protected maxRequestsPerCrawl?: number;
|
|
433
|
-
protected handledRequestsCount: number;
|
|
434
|
-
protected statusMessageLoggingInterval: number;
|
|
435
|
-
protected statusMessageCallback?: StatusMessageCallback;
|
|
436
|
-
protected sessionPoolOptions: SessionPoolOptions;
|
|
437
|
-
protected useSessionPool: boolean;
|
|
438
|
-
protected autoscaledPoolOptions: AutoscaledPoolOptions;
|
|
439
|
-
protected httpClient: BaseHttpClient;
|
|
440
|
-
protected retryOnBlocked: boolean;
|
|
441
|
-
protected respectRobotsTxtFile: boolean | {
|
|
442
|
-
userAgent?: string;
|
|
443
|
-
};
|
|
444
|
-
protected onSkippedRequest?: SkippedRequestCallback;
|
|
445
|
-
private _closeEvents?;
|
|
446
|
-
private shouldLogMaxProcessedRequestsExceeded;
|
|
447
|
-
private shouldLogMaxEnqueuedRequestsExceeded;
|
|
448
|
-
private experiments;
|
|
449
|
-
private readonly robotsTxtFileCache;
|
|
450
|
-
private _experimentWarnings;
|
|
451
|
-
private readonly crawlerId;
|
|
452
|
-
private readonly hasExplicitId;
|
|
447
|
+
/** Whether a `run()` on this instance has already finished - a repeated one continues where it left off. */
|
|
448
|
+
get hasFinishedBefore(): boolean;
|
|
449
|
+
get log(): CrawleeLogger;
|
|
450
|
+
protected readonly requestHandler: RequestHandler<ExtendedContext>;
|
|
451
|
+
protected readonly internalTimeoutMillis: number;
|
|
452
|
+
protected blockedStatusCodes: Set<number>;
|
|
453
|
+
protected readonly additionalHttpErrorStatusCodes: Set<number>;
|
|
454
|
+
protected readonly httpClient: BaseHttpClient;
|
|
455
|
+
protected readonly retryOnBlocked: boolean;
|
|
456
|
+
/**
|
|
457
|
+
* @internal
|
|
458
|
+
*/
|
|
453
459
|
protected static optionsShape: {
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
498
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
499
|
-
httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
500
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
501
|
-
configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
502
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
503
|
-
storageClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
504
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
505
|
-
eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
506
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
507
|
-
minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
508
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
509
|
-
maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
510
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
511
|
-
maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
512
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
513
|
-
keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
514
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
515
|
-
log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
516
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
517
|
-
experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
518
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
519
|
-
statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
520
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
521
|
-
id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
|
|
460
|
+
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
461
|
+
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
462
|
+
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
463
|
+
requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
464
|
+
requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
465
|
+
requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
466
|
+
requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
467
|
+
errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
468
|
+
failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
469
|
+
maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
470
|
+
sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
471
|
+
maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
472
|
+
maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
473
|
+
taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
474
|
+
concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
475
|
+
sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
476
|
+
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
477
|
+
statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
478
|
+
statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
479
|
+
additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
480
|
+
ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
481
|
+
blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
482
|
+
retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
|
|
483
|
+
respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
|
|
484
|
+
transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
|
|
485
|
+
requestQueue: z.ZodOptional<z.ZodEnum<{
|
|
486
|
+
deferred: "deferred";
|
|
487
|
+
writeThrough: "writeThrough";
|
|
488
|
+
}>>;
|
|
489
|
+
}, z.core.$strict>]>>;
|
|
490
|
+
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
491
|
+
httpClient: z.ZodOptional<z.ZodInstanceOf<BaseHttpClient>>;
|
|
492
|
+
configuration: z.ZodOptional<z.ZodInstanceOf<Configuration>>;
|
|
493
|
+
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
494
|
+
eventManager: z.ZodOptional<z.ZodInstanceOf<EventManager>>;
|
|
495
|
+
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
496
|
+
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
497
|
+
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
498
|
+
initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
499
|
+
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
500
|
+
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
501
|
+
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
502
|
+
id: z.ZodOptional<z.ZodString>;
|
|
522
503
|
};
|
|
523
504
|
/**
|
|
524
505
|
* All `BasicCrawler` parameters are passed via an options object.
|
|
525
506
|
*/
|
|
526
|
-
constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext> & RequireContextPipeline<CrawlingContext, Context>);
|
|
507
|
+
constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & RequireContextPipeline<CrawlingContext, Context>);
|
|
508
|
+
/**
|
|
509
|
+
* Builds the crawler-owned default {@link ConcurrencySystem} from the resolved
|
|
510
|
+
* `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
|
|
511
|
+
* {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} was injected.
|
|
512
|
+
*
|
|
513
|
+
* Subclasses may override this to tune the default system (e.g. {@link HttpCrawler} raises the starting
|
|
514
|
+
* concurrency and relaxes the event loop signal) while still honouring the user's shortcuts.
|
|
515
|
+
*/
|
|
516
|
+
protected createDefaultConcurrencySystem(options: ConcurrencySystemOptions): ConcurrencySystem;
|
|
517
|
+
/**
|
|
518
|
+
* Determines if the given HTTP status code is an error status code given
|
|
519
|
+
* the default behaviour and user-set preferences.
|
|
520
|
+
* @param status
|
|
521
|
+
* @returns `true` if the status code is considered an error, `false` otherwise
|
|
522
|
+
*/
|
|
523
|
+
protected isErrorStatusCode(status: number): boolean;
|
|
527
524
|
/**
|
|
528
525
|
* Checks if the given error is a proxy error by comparing its message to a list of known proxy error messages.
|
|
529
526
|
* Used for retrying requests that failed due to proxy errors.
|
|
@@ -532,21 +529,31 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
|
|
|
532
529
|
*/
|
|
533
530
|
protected isProxyError(error: Error): boolean;
|
|
534
531
|
/**
|
|
532
|
+
* Sets the status message for the current crawler run.
|
|
533
|
+
*
|
|
535
534
|
* This method is periodically called by the crawler, every `statusMessageLoggingInterval` seconds.
|
|
535
|
+
*
|
|
536
|
+
* The message is logged and broadcast via the {@link EventType.STATUS_MESSAGE|`statusMessage`}
|
|
537
|
+
* event. Integrations such as the Apify SDK subscribe to that event and forward the message to
|
|
538
|
+
* their status-reporting backend (e.g. the Apify platform).
|
|
536
539
|
*/
|
|
537
|
-
setStatusMessage(message: string, options?: SetStatusMessageOptions):
|
|
538
|
-
private getPeriodicLogger;
|
|
540
|
+
setStatusMessage(message: string, options?: SetStatusMessageOptions): void;
|
|
539
541
|
/**
|
|
540
|
-
* Runs the crawler. Returns a promise that resolves once
|
|
541
|
-
*
|
|
542
|
+
* Runs the crawler. Returns a promise that resolves once every request has been processed and the crawler's
|
|
543
|
+
* finished-check ({@link BasicCrawlerOptions.taskLoopOptions|`taskLoopOptions.isFinishedFunction`}, or the
|
|
544
|
+
* default "the request manager is empty") reports that the crawl is over.
|
|
542
545
|
*
|
|
543
546
|
* We can use the `requests` parameter to enqueue the initial requests — it is a shortcut for
|
|
544
547
|
* running {@link BasicCrawler.addRequests|`crawler.addRequests()`} before {@link BasicCrawler.run|`crawler.run()`}.
|
|
545
548
|
*
|
|
549
|
+
* Calling `run()` again on the same instance keeps crawling the same request manager - requests the previous
|
|
550
|
+
* run handled (a failed one counts as handled) are not processed again. Purge the queue or open a fresh one
|
|
551
|
+
* if that is what you want.
|
|
552
|
+
*
|
|
546
553
|
* @param [requests] The requests to add.
|
|
547
|
-
* @param [options] Options for the
|
|
554
|
+
* @param [options] Options for adding the initial requests.
|
|
548
555
|
*/
|
|
549
|
-
run(requests?:
|
|
556
|
+
run(requests?: TypedRequestsLike<Routes>, options?: CrawlerRunOptions): Promise<FinalStatistics>;
|
|
550
557
|
/**
|
|
551
558
|
* Gracefully stops the current run of the crawler.
|
|
552
559
|
*
|
|
@@ -554,32 +561,57 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
|
|
|
554
561
|
*
|
|
555
562
|
* To stop the crawler immediately, use {@link BasicCrawler.teardown|`crawler.teardown()`} instead.
|
|
556
563
|
*/
|
|
557
|
-
stop(
|
|
558
|
-
|
|
564
|
+
stop(reason?: string): void;
|
|
565
|
+
/**
|
|
566
|
+
* Stops dispatching new requests, letting the in-progress ones finish. Resolves once they have settled, or rejects
|
|
567
|
+
* after `timeoutSecs` if they take too long. Unlike {@link BasicCrawler.stop|`stop()`}, this does not end the
|
|
568
|
+
* run — {@link BasicCrawler.run|`run()`} stays pending until {@link BasicCrawler.resume|`resume()`}.
|
|
569
|
+
*
|
|
570
|
+
* > *NOTE:* The {@link BasicCrawler.concurrencySystem|concurrency system} keeps monitoring and autoscaling
|
|
571
|
+
* throughout, since a shared one may still be serving other crawlers.
|
|
572
|
+
*/
|
|
573
|
+
pause(timeoutSecs?: number): Promise<void>;
|
|
574
|
+
/**
|
|
575
|
+
* Resumes a run suspended with {@link BasicCrawler.pause|`pause()`}, letting the crawler dispatch requests
|
|
576
|
+
* again. A no-op on a crawler that is not paused.
|
|
577
|
+
*/
|
|
578
|
+
resume(): void;
|
|
579
|
+
/**
|
|
580
|
+
* Returns the crawler's {@link IRequestManager|request manager}, opening the default {@link RequestQueue}
|
|
581
|
+
* if none has been configured or opened yet.
|
|
582
|
+
*/
|
|
583
|
+
getRequestManager(): Promise<IRequestManager>;
|
|
584
|
+
/**
|
|
585
|
+
* @deprecated Use {@link BasicCrawler.getRequestManager|`getRequestManager()`} instead. This returns the
|
|
586
|
+
* crawler's request manager, which is no longer guaranteed to be a {@link RequestQueue}.
|
|
587
|
+
*/
|
|
588
|
+
getRequestQueue(): Promise<IRequestManager>;
|
|
559
589
|
useState<State extends Dictionary = Dictionary>(defaultValue?: State): Promise<State>;
|
|
560
|
-
protected get pendingRequestCountApproximation(): number;
|
|
561
|
-
protected calculateEnqueuedRequestLimit(explicitLimit?: number): number | undefined;
|
|
562
|
-
protected handleSkippedRequest(options: Parameters<SkippedRequestCallback>[0]): Promise<void>;
|
|
563
590
|
/**
|
|
564
591
|
* Adds requests to the queue in batches. By default, it will resolve after the initial batch is added, and continue
|
|
565
592
|
* adding the rest in background. You can configure the batch size via `batchSize` option and the sleep time in between
|
|
566
593
|
* the batches via `waitBetweenBatchesMillis`. If you want to wait for all batches to be added to the queue, you can use
|
|
567
594
|
* the `waitForAllRequestsToBeAdded` promise you get in the response object.
|
|
568
595
|
*
|
|
596
|
+
* Optionally, the requests can be filtered using `include`/`exclude` glob or regexp patterns and an
|
|
597
|
+
* enqueue `strategy` (both AND-ed together, same as {@link CrawlingContext.enqueueLinks|`enqueueLinks`}),
|
|
598
|
+
* relative to `baseUrl`. Unlike `enqueueLinks`, there is no implicit "current page" to anchor the strategy
|
|
599
|
+
* to, so `strategy` defaults to {@link EnqueueStrategy.All|`all`} here.
|
|
600
|
+
*
|
|
569
601
|
* This is an alias for calling `addRequestsBatched()` on the implicit `RequestQueue` for this crawler instance.
|
|
570
602
|
*
|
|
571
603
|
* @param requests The requests to add
|
|
572
604
|
* @param options Options for the request queue
|
|
573
605
|
*/
|
|
574
|
-
addRequests(requests: ReadonlyDeep<
|
|
606
|
+
addRequests(requests: ReadonlyDeep<TypedRequestsLike<Routes>>, options?: CrawlerAddRequestsOptions): Promise<CrawlerAddRequestsResult>;
|
|
575
607
|
/**
|
|
576
608
|
* Pushes data to the specified {@link Dataset}, or the default crawler {@link Dataset} by calling {@link Dataset.pushData}.
|
|
577
609
|
*/
|
|
578
|
-
pushData(data: Parameters<Dataset['pushData']>[0],
|
|
610
|
+
pushData(data: Parameters<Dataset['pushData']>[0], datasetIdentifier?: string | StorageIdentifier): Promise<void>;
|
|
579
611
|
/**
|
|
580
612
|
* Retrieves the specified {@link Dataset}, or the default crawler {@link Dataset}.
|
|
581
613
|
*/
|
|
582
|
-
getDataset(
|
|
614
|
+
getDataset(identifier?: string | StorageIdentifier): Promise<Dataset>;
|
|
583
615
|
/**
|
|
584
616
|
* Retrieves data from the default crawler {@link Dataset} by calling {@link Dataset.getData}.
|
|
585
617
|
*/
|
|
@@ -592,112 +624,76 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
|
|
|
592
624
|
/**
|
|
593
625
|
* Initializes the crawler.
|
|
594
626
|
*/
|
|
595
|
-
protected
|
|
596
|
-
protected runRequestHandler(crawlingContext: CrawlingContext): Promise<void>;
|
|
597
|
-
/**
|
|
598
|
-
* Handles blocked request
|
|
599
|
-
*/
|
|
600
|
-
protected _throwOnBlockedRequest(session: Session, statusCode: number): void;
|
|
601
|
-
private isAllowedBasedOnRobotsTxtFile;
|
|
602
|
-
protected getRobotsTxtFileForUrl(url: string): Promise<RobotsTxtFile | undefined>;
|
|
603
|
-
protected _pauseOnMigration(): Promise<void>;
|
|
604
|
-
/**
|
|
605
|
-
* Initializes the RequestManager based on the configured requestList and requestQueue.
|
|
606
|
-
*/
|
|
607
|
-
private initializeRequestManager;
|
|
627
|
+
protected init(): Promise<void>;
|
|
608
628
|
/**
|
|
609
|
-
*
|
|
629
|
+
* The navigation timeout (pre-navigation hooks, navigation, and post-navigation hooks) in milliseconds, used
|
|
630
|
+
* to size the internal request timeout. `BasicCrawler` has no navigation phase, so this is 0; the HTTP and
|
|
631
|
+
* browser crawlers override it with their `navigationTimeoutSecs`.
|
|
610
632
|
*/
|
|
611
|
-
protected
|
|
633
|
+
protected getNavigationTimeoutMillis(): number;
|
|
634
|
+
protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
|
|
612
635
|
/**
|
|
613
|
-
*
|
|
614
|
-
*
|
|
615
|
-
*
|
|
636
|
+
* Runs `callback` inside a {@link StorageTransaction}, unless transactional storage is disabled.
|
|
637
|
+
* Deliberately does **not** commit on return - the pipeline hands failures to `handleRequestFailure`, so a
|
|
638
|
+
* normal return says nothing about success. `handleRequest` commits on success.
|
|
616
639
|
*/
|
|
617
|
-
|
|
640
|
+
private runInStorageTransaction;
|
|
618
641
|
/**
|
|
619
|
-
*
|
|
620
|
-
* then retries them in a case of an error, etc.
|
|
642
|
+
* Handles blocked request
|
|
621
643
|
*/
|
|
622
|
-
protected
|
|
644
|
+
protected throwOnBlockedRequest(statusCode: number): void;
|
|
623
645
|
/**
|
|
624
|
-
*
|
|
625
|
-
*
|
|
626
|
-
*
|
|
627
|
-
*
|
|
628
|
-
*
|
|
646
|
+
* Records an HTTP 429 against the URL's domain so the request manager can hold the retry back.
|
|
647
|
+
*
|
|
648
|
+
* @param retryAfterHeader The raw `Retry-After` response header, if the server sent one.
|
|
649
|
+
* @returns `true` if the manager took responsibility for the delay, in which case the caller should throw
|
|
650
|
+
* {@link RequestThrottledError} rather than treating the response as a blocked session.
|
|
629
651
|
*/
|
|
630
|
-
protected
|
|
652
|
+
protected recordDomainRateLimit(url: string, retryAfterHeader?: string | null): boolean;
|
|
653
|
+
/** @internal */
|
|
654
|
+
protected getRobotsTxtFileForUrl(url: string): Promise<RobotsTxtFile | undefined>;
|
|
655
|
+
/** Runs the request handler and marks the request as handled. Failures are left to `handleRequestFailure`. */
|
|
656
|
+
private handleRequest;
|
|
631
657
|
/**
|
|
632
658
|
* Generator function that yields requests injected with the given crawl depth.
|
|
633
659
|
* @internal
|
|
634
660
|
*/
|
|
635
661
|
protected addCrawlDepthRequestGenerator(requests: RequestsLike, newRequestDepth: number): AsyncGenerator<Source, void, undefined>;
|
|
636
|
-
/**
|
|
637
|
-
* Run async callback with given timeout and retry. Returns the result of the callback.
|
|
638
|
-
* @ignore
|
|
639
|
-
*/
|
|
640
|
-
protected _timeoutAndRetry<T>(handler: () => Promise<T>, timeout: number, error: Error | string, maxRetries?: number, retried?: number): Promise<T>;
|
|
641
|
-
/**
|
|
642
|
-
* Returns true if either RequestList or RequestQueue have a request ready for processing.
|
|
643
|
-
*/
|
|
644
|
-
protected _isTaskReadyFunction(): Promise<boolean>;
|
|
645
|
-
/**
|
|
646
|
-
* Returns true if both RequestList and RequestQueue have all requests finished.
|
|
647
|
-
*/
|
|
648
|
-
protected _defaultIsFinishedFunction(): Promise<boolean>;
|
|
649
|
-
private _rotateSession;
|
|
650
|
-
/**
|
|
651
|
-
* Unwraps errors thrown by the context pipeline to get the actual user error.
|
|
652
|
-
* RequestHandlerError and ContextPipelineInitializationError wrap the actual error.
|
|
653
|
-
*/
|
|
654
|
-
private unwrapError;
|
|
655
662
|
/**
|
|
656
663
|
* Handles errors thrown by user provided requestHandler()
|
|
657
664
|
*/
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
protected _handleFailedRequestHandler(crawlingContext: CrawlingContext, error: Error): Promise<void>;
|
|
665
|
+
private requestFunctionErrorHandler;
|
|
666
|
+
private handleFailedRequestHandler;
|
|
661
667
|
/**
|
|
662
668
|
* Resolves the most verbose error message from a thrown error
|
|
663
669
|
* @param error The error received
|
|
664
670
|
* @returns The message to be logged
|
|
665
671
|
*/
|
|
666
|
-
protected
|
|
667
|
-
protected _canRequestBeRetried(request: Request, error: Error): boolean;
|
|
672
|
+
protected getMessageFromError(error: Error, forceStack?: boolean): string;
|
|
668
673
|
/**
|
|
669
|
-
*
|
|
670
|
-
*/
|
|
671
|
-
protected _loadHandledRequestCount(): Promise<void>;
|
|
672
|
-
protected _executeHooks<HookLike extends (...args: any[]) => Awaitable<void>>(hooks: HookLike[], ...args: Parameters<HookLike>): Promise<void>;
|
|
673
|
-
/**
|
|
674
|
-
* Stops the crawler immediately.
|
|
674
|
+
* Ends the run in progress immediately, without waiting for the requests already in flight.
|
|
675
675
|
*
|
|
676
|
-
* This
|
|
676
|
+
* This runs at the end of every {@link BasicCrawler.run|`run()`}, so what it releases is per-run only, and
|
|
677
|
+
* a repeated `run()` has to find everything it needs either intact or re-establishable. What outlives a run is
|
|
678
|
+
* released by {@link BasicCrawler.destroy|`destroy()`} instead.
|
|
677
679
|
*
|
|
678
680
|
* To stop the crawler gracefully (waiting for all running requests to finish), use {@link BasicCrawler.stop|`crawler.stop()`} instead.
|
|
679
681
|
*/
|
|
680
682
|
teardown(): Promise<void>;
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
683
|
+
/**
|
|
684
|
+
* Releases what the crawler owns beyond a single run — in the browser crawlers, the browser pool. A finished
|
|
685
|
+
* `run()` has already released everything a run owns, so this is only needed before dropping a crawler you
|
|
686
|
+
* could otherwise have run again.
|
|
687
|
+
*/
|
|
688
|
+
destroy(): Promise<void>;
|
|
689
|
+
[Symbol.asyncDispose](): Promise<void>;
|
|
690
|
+
protected getCookieHeaderFromRequest(request: Request): string;
|
|
689
691
|
}
|
|
690
|
-
export interface CrawlerAddRequestsOptions extends AddRequestsBatchedOptions {
|
|
692
|
+
export interface CrawlerAddRequestsOptions extends AddRequestsBatchedOptions, EnqueueUrlsOptions {
|
|
691
693
|
}
|
|
692
694
|
export interface CrawlerAddRequestsResult extends AddRequestsBatchedResult {
|
|
693
695
|
}
|
|
694
696
|
export interface CrawlerRunOptions extends CrawlerAddRequestsOptions {
|
|
695
|
-
/**
|
|
696
|
-
* Whether to purge the RequestQueue before running the crawler again. Defaults to true, so it is possible to reprocess failed requests.
|
|
697
|
-
* When disabled, only new requests will be considered. Note that even a failed request is considered as handled.
|
|
698
|
-
* @default true
|
|
699
|
-
*/
|
|
700
|
-
purgeRequestQueue?: boolean;
|
|
701
697
|
}
|
|
702
698
|
/**
|
|
703
699
|
* Creates new {@link Router} instance that works based on request labels.
|
|
@@ -723,5 +719,5 @@ export interface CrawlerRunOptions extends CrawlerAddRequestsOptions {
|
|
|
723
719
|
* await crawler.run();
|
|
724
720
|
* ```
|
|
725
721
|
*/
|
|
726
|
-
export declare function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext,
|
|
727
|
-
|
|
722
|
+
export declare function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
|
|
723
|
+
export declare function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
|